From 989cbc41a7d30e9037b67c4a0d15c5bebcab5f86 Mon Sep 17 00:00:00 2001 From: Andrew Leonard <31470007+andrew-m-leonard@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:46:01 +0000 Subject: [PATCH 01/15] Updates to comparable_patch.sh to support Windows vendor neutral comparison (#3536) * Add Windows comparable patch fixes Signed-off-by: U-andrew-repro-te\adoptium * Add Windows comparable patch fixes Signed-off-by: U-andrew-repro-te\adoptium * Add Windows comparable patch fixes Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium * Update doc Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium * Fix linter Signed-off-by: U-andrew-repro-te\adoptium --------- Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 69 +++++++++- tooling/comparable_patch.sh | 252 ++++++++++++++++++++++++++++------ 2 files changed, 281 insertions(+), 40 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index 6f565da6a..bf930e86d 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -39,7 +39,7 @@ remove any Signatures from the executeables. ## Comparable Build Tools -1. comparable_patch.sh : Patches a JDK folder to enable "Comparable" comparison of two different Vendor JDK builds. +comparable_patch.sh : Patches a JDK folder to enable "Comparable" comparison of two different Vendor JDK builds. The patching process involves: - Expanding all zips and jmods, so executeables can be processed to remove signatures prior to comaprison. @@ -47,4 +47,71 @@ The patching process involves: - Neutralise VS_VERSION_INFO Vendor strings (Windows). - Remove non-comparable CRC generated uuid values, which are binary values based on the hash of the content (Windows & MacOS). - Remove Vendor strings embedded in executables, classes and text files. +- Remove module-info differences due to "hash" of Signed module executables - Remove any non-deterministic build process artifact strings, like Manifest Created-By stamps. + +### How to setup and run comparable_patch.sh on Windows + +#### Tooling setup: + +1. The comparable patch tools, tooling/src/c/WindowsUpdateVsVersionInfo.c and src/java/temurin/tools/BinRepl.java need compiling +before the comparable_patch.sh can be run + +2. Compile tooling/src/c/WindowsUpdateVsVersionInfo.c : + +- Ensure VS2022 SDK is installed and on PATH +- Compile: + - cd tooling/src/c + - cl WindowsUpdateVsVersionInfo.c version.lib + +3. Compile src/java/temurin/tools/BinRepl.java : + +- Ensure suitable JDK on PATH +- cd tooling/src/java +- javac temurin/tools/BinRepl.java + +4. Setting environment within a CYGWIN shell : + +- For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH +- For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH +- For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra\~1/micros\~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH +- For running BinRepl java : export PATH=/bin:$PATH + +#### Running comparable_patch.sh: + +1. Unzip your JDK archive into a directory (eg.jdk1) + +2. Run comparable_patch.sh + +```bash +bash comparable_patch.sh +``` + +The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": + +```java +java -XshowSettings: +... + java.vendor = Eclipse Adoptium + java.vendor.url = https://adoptium.net/ + java.vendor.url.bug = https://github.com/adoptium/adoptium-support/issues + java.vendor.version = Temurin-21.0.1+12 +... +``` + +eg. + +```bash +bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues" +``` + +3. Unzip the other Vendor JDK to compare with, say into "jdk2", and run a similar comparable_patch.sh +for that Vendor branding + +4. Diff recursively the now Vendor neutralized jdk directories + +```bash +diff -r jdk1 jdk2 +``` + +The diff should be "identical" if the two Vendor JDK's are "Comparable", ie."Identical except for the Vendor Branding" diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh index 88cef75ba..52003b1ba 100755 --- a/tooling/comparable_patch.sh +++ b/tooling/comparable_patch.sh @@ -28,6 +28,11 @@ set -eu TEMURIN_TOOLS_BINREPL="temurin.tools.BinRepl" +if [ "$#" -ne 6 ]; then + echo "Syntax: comparable_patch.sh " + exit 1 +fi + JDK_DIR="$1" VERSION_REPL="$2" VENDOR_NAME="$3" @@ -36,12 +41,12 @@ VENDOR_BUG_URL="$5" VENDOR_VM_BUG_URL="$6" # Remove excluded files known to differ +# NOTICE - Vendor specfic notice text file +# cacerts - Vendors use different cacerts +# classes.jsa, classes_nocoops.jsa - CDS archive caches will differ due to Vendor string differences function removeExcludedFiles() { - if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class" - else - excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa" - fi + excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa" + echo "Removing excluded files known to differ: ${excluded}" for exclude in $excluded do @@ -53,36 +58,183 @@ function removeExcludedFiles() { done done + echo "Successfully removed all excluded files from ${JDK_DIR}" +} + +# Normalize the following ModuleAttributes that can be ordered differently +# depending on how the vendor has signed and re-packed the JMODs +# - ModuleResolution: +# - ModuleTarget: +# java.base also requires the dependent module "hash:" values to be excluded +# as they differ due to the Signatures +function processModuleInfo() { if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - echo "Removing java.base module-info.class, known to differ by jdk.jpackage module hash" - rm "${JDK_DIR}/jmods/expanded_java.base.jmod/classes/module-info.class" - rm "${JDK_DIR}/lib/modules_extracted/java.base/module-info.class" + echo "Normalizing ModuleAttributes order in module-info.class, converting to javap" + + moduleAttr="ModuleResolution ModuleTarget" + + FILES=$(find "${JDK_DIR}" -type f -name "module-info.class") + for f in $FILES + do + echo "javap and re-order ModuleAttributes for $f" + javap -v -sysinfo -l -p -c -s -constants "$f" > "$f.javap.tmp" + rm "$f" + + cc=99 + foundAttr=false + attrName="" + # Clear any attr tmp files + for attr in $moduleAttr + do + rm -f "$f.javap.$attr" + done + + while IFS= read -r line + do + cc=$((cc+1)) + + # Module attr have only 1 line definition + if [[ "$foundAttr" = true ]] && [[ "$cc" -gt 1 ]]; then + foundAttr=false + attrName="" + fi + + # If not processing an attr then check for attr + if [[ "$foundAttr" = false ]]; then + for attr in $moduleAttr + do + if [[ "$line" =~ .*"$attr:".* ]]; then + cc=0 + foundAttr=true + attrName="$attr" + fi + done + fi + + # Echo attr to attr tmp file, otherwise to tmp2 + if [[ "$foundAttr" = true ]]; then + echo "$line" >> "$f.javap.$attrName" + else + echo "$line" >> "$f.javap.tmp2" + fi + done < "$f.javap.tmp" + rm "$f.javap.tmp" + + # Remove javap Classfile and timestamp and SHA-256 hash + if [[ "$f" =~ .*"java.base".* ]]; then + grep -v "Last modified\|Classfile\|SHA-256 checksum\|hash:" "$f.javap.tmp2" > "$f.javap" + else + grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" + fi + rm "$f.javap.tmp2" + + # Append any ModuleAttr tmp files + for attr in $moduleAttr + do + if [[ -f "$f.javap.$attr" ]]; then + cat "$f.javap.$attr" >> "$f.javap" + fi + rm -f "$f.javap.$attr" + done + done fi - echo "Successfully removed all excluded files from ${JDK_DIR}" +} + +# Process SystemModules classes to remove ModuleHashes$Builder differences due to Signatures +# 1. javap +# 2. search for line: // Method jdk/internal/module/ModuleHashes$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes$Builder; +# 3. followed 3 lines later by: // String +# 4. then remove all lines until next: invokevirtual +# 5. remove Last modified, Classfile and SHA-256 checksum javap artefact statements +function removeSystemModulesHashBuilderParams() { + # Key strings + moduleHashesFunction="// Method jdk/internal/module/ModuleHashes\$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes\$Builder;" + moduleString="// String " + virtualFunction="invokevirtual" + + systemModules="SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class" + echo "Removing SystemModules ModulesHashes\$Builder differences" + for systemModule in $systemModules + do + FILES=$(find "${JDK_DIR}" -type f -name "$systemModule") + for f in $FILES + do + echo "Processing $f" + javap -v -sysinfo -l -p -c -s -constants "$f" > "$f.javap.tmp" + rm "$f" + + # Remove "instruction number:" prefix, so we can just match code + sed -i -E "s/^[[:space:]]+[0-9]+:(.*)/\1/" "$f.javap.tmp" + + cc=99 + found=false + while IFS= read -r line + do + cc=$((cc+1)) + # Detect hashForModule function + if [[ "$line" =~ .*"$moduleHashesFunction".* ]]; then + cc=0 + fi + # 3rd instruction line is the Module string to confirm entry + if [[ "$cc" -eq 3 ]] && [[ "$line" =~ .*"$moduleString"[a-z\.]+.* ]]; then + found=true + module=$(echo "$line" | tr -s ' ' | tr -d '\r' | cut -d' ' -f6) + echo "==> Found $module ModuleHashes\$Builder function, skipping hash parameter" + fi + # hasForModule function section finishes upon finding invokevirtual + if [[ "$found" = true ]] && [[ "$line" =~ .*"$virtualFunction".* ]]; then + found=false + fi + if [[ "$found" = false ]]; then + echo "$line" >> "$f.javap.tmp2" + fi + done < "$f.javap.tmp" + rm "$f.javap.tmp" + grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" + rm "$f.javap.tmp2" + done + done + + echo "Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}" } # Remove the Windows EXE/DLL timestamps and internal VS CRC and debug repro hex values +# The Windows PE format contains various values determined from the binary content +# which will vary due to the different Vendor branding +# timestamp - Used to be an actual timestamp but MSFT changed this to a checksum determined from binary content +# checksum - A checksum value of the binary +# reprohex - A hex UUID to identify the binary version, again generated from binary content function removeWindowsNonComparableData() { echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}" FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll') for f in $FILES do echo "Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f" - rm -f dumpbin.tmp - if ! dumpbin "$f" /ALL > dumpbin.tmp; then - echo " FAILED == > dumpbin \"$f\" /ALL > dumpbin.tmp" + + # Determine non-comparable data using dumpbin + dmpfile="$f.dumpbin.tmp" + rm -f "$dmpfile" + if ! dumpbin "$f" /ALL > "$dmpfile"; then + echo " FAILED == > dumpbin \"$f\" /ALL > $dmpfile" exit 1 fi - timestamp=$(grep "time date stamp" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f2) - checksum=$(grep "checksum" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f2) - reprohex=$(grep "${timestamp} repro" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r') - reprohexhalf=$(grep "${timestamp} repro" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r') + + # Determine non-comparable stamps and hex codes from dumpbin output + timestamp=$(grep "time date stamp" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2) + checksum=$(grep "checksum" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2) + reprohex=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r') + reprohexhalf=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r') + rm -f "$dmpfile" + + # Neutralize reprohex string if [ -n "$reprohex" ]; then if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA"; then echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\"" exit 1 fi fi + + # Neutralize timestamp hex string hexstr="00000000" timestamphex=${hexstr:0:-${#timestamp}}$timestamp timestamphexLE="${timestamphex:6:2}:${timestamphex:4:2}:${timestamphex:2:2}:${timestamphex:0:2}" @@ -97,6 +249,7 @@ function removeWindowsNonComparableData() { fi fi + # Neutralize checksum string # Prefix checksum to 8 digits hexstr="00000000" checksumhex=${hexstr:0:-${#checksum}}$checksum @@ -136,7 +289,7 @@ function removeMacOSNonComparableData() { echo "Successfully removed all MacOS dylib non-comparable UUID from ${JDK_DIR}" } -# Neutralize Windows VS_VERSION_INFO CompanyName +# Neutralize Windows VS_VERSION_INFO CompanyName from the resource compiled PE section function neutraliseVsVersionInfo() { echo "Updating EXE/DLL VS_VERSION_INFO in ${JDK_DIR}" FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll') @@ -231,38 +384,53 @@ function neutraliseReleaseFile() { echo "Removing Vendor strings from release file ${JDK_DIR}/release" if [[ "$OS" =~ Darwin* ]]; then + # Remove Vendor versions sed -i "" "s=$VERSION_REPL==g" "${JDK_DIR}/release" sed -i "" "s=$VENDOR_NAME==g" "${JDK_DIR}/release" - # BUILD_INFO likely different since built on different machines - sed -i "" "s=^BUILD_INFO.*$==g" "${JDK_DIR}/release" - - # BUILD_SOURCE possibly built not using temurin build scripts - sed -i "" "s=^BUILD_SOURCE.*$==g" "${JDK_DIR}/release" - sed -i "" "s=^BUILD_SOURCE_REPO.*$==g" "${JDK_DIR}/release" - - # Temurin JVM_VARIANT has capital "Hotspot" - sed -i "" "s,^JVM_VARIANT=\"Hotspot\",JVM_VARIANT=\"hotspot\",g" "${JDK_DIR}/release" + # Temurin BUILD_* likely different since built on different machines and bespoke to Temurin + sed -i "" "/^BUILD_INFO/d" "${JDK_DIR}/release" + sed -i "" "/^BUILD_SOURCE/d" "${JDK_DIR}/release" + sed -i "" "/^BUILD_SOURCE_REPO/d" "${JDK_DIR}/release" + + # Remove bespoke Temurin fields + sed -i "" "/^SOURCE/d" "${JDK_DIR}/release" + sed -i "" "/^FULL_VERSION/d" "${JDK_DIR}/release" + sed -i "" "/^SEMANTIC_VERSION/d" "${JDK_DIR}/release" + sed -i "" "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "" "/^JVM_VERSION/d" "${JDK_DIR}/release" + sed -i "" "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "" "/^IMAGE_TYPE/d" "${JDK_DIR}/release" else + # Remove Vendor versions sed -i "s=$VERSION_REPL==g" "${JDK_DIR}/release" sed -i "s=$VENDOR_NAME==g" "${JDK_DIR}/release" - # BUILD_INFO likely different since built on different machines - sed -i "s=^BUILD_INFO.*$==g" "${JDK_DIR}/release" - - # BUILD_SOURCE possibly built not using temurin build scripts - sed -i "s=^BUILD_SOURCE.*$==g" "${JDK_DIR}/release" - sed -i "s=^BUILD_SOURCE_REPO.*$==g" "${JDK_DIR}/release" - - # Temurin JVM_VARIANT has capital "Hotspot" - sed -i "s,^JVM_VARIANT=\"Hotspot\",JVM_VARIANT=\"hotspot\",g" "${JDK_DIR}/release" + # Temurin BUILD_* likely different since built on different machines and bespoke to Temurin + sed -i "/^BUILD_INFO/d" "${JDK_DIR}/release" + sed -i "/^BUILD_SOURCE/d" "${JDK_DIR}/release" + sed -i "/^BUILD_SOURCE_REPO/d" "${JDK_DIR}/release" + + # Remove bespoke Temurin fields + sed -i "/^SOURCE/d" "${JDK_DIR}/release" + sed -i "/^FULL_VERSION/d" "${JDK_DIR}/release" + sed -i "/^SEMANTIC_VERSION/d" "${JDK_DIR}/release" + sed -i "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "/^JVM_VERSION/d" "${JDK_DIR}/release" + sed -i "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "/^IMAGE_TYPE/d" "${JDK_DIR}/release" fi } -if [ "$#" -ne 6 ]; then - echo "Syntax: cmd " - exit 1 -fi +# Remove some non-JDK files that some Vendors distribute +# - NEWS : Some Vendors provide a NEWS text file +# - demo : Not all vendors distribute the demo examples +function removeNonJdkFiles() { + echo "Removing non-JDK files" + + rm -f "${JDK_DIR}/NEWS" + rm -rf "${JDK_DIR}/demo" +} if [ ! -d "${JDK_DIR}" ]; then echo "$JDK_DIR does not exist" @@ -299,6 +467,10 @@ echo "Successfully removed all Signatures from ${JDK_DIR}" removeExcludedFiles +processModuleInfo + +removeSystemModulesHashBuilderParams + if [[ "$OS" =~ CYGWIN* ]]; then neutraliseVsVersionInfo fi @@ -319,6 +491,8 @@ neutraliseManifests neutraliseReleaseFile +removeNonJdkFiles + echo "***********" echo "SUCCESS :-)" echo "***********" From 32021ac5b882c1e00e1550f1d5e7a149d9f143bb Mon Sep 17 00:00:00 2001 From: George Adams Date: Tue, 21 Nov 2023 11:55:35 +0100 Subject: [PATCH 02/15] pin JDK11 x64 build to XCode rather than commandline tools (#3541) * pin JDK11 x64 build to XCode rather than commandline tools * update actions --- .github/workflows/build.yml | 27 ++++++++++++------- .../platform-specific-configurations/mac.sh | 5 ++-- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 13cd50c9f..887f28531 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,12 +109,15 @@ jobs: build_macos: name: macOS - runs-on: macos-11 + runs-on: ${{ matrix.version.distro }} strategy: fail-fast: false matrix: os: [macOS] - version: [jdk8u, jdk11u] + version: [ + { name: jdk8u, distro: macos-11 }, + { name: jdk11u, distro: macos-13 }, + ] variant: [temurin] steps: @@ -129,19 +132,25 @@ jobs: with: java-version: 7 distribution: 'zulu' - if: matrix.version == 'jdk8u' + if: matrix.version.name == 'jdk8u' - - name: Select correct Xcode + - name: Select correct Xcode (JDK8) + if: matrix.version.name == 'jdk8u' run: | rm -rf /Applications/Xcode.app ln -s /Applications/Xcode_11.7.app /Applications/Xcode.app - ln -s /Applications/Xcode_11.7.app /Applications/Xcode-11.7.app + + - name: Select correct Xcode (JDK11+) + if: matrix.version.name != 'jdk8u' + run: | + rm -rf /Applications/Xcode.app + ln -s /Applications/Xcode_15.0.1.app /Applications/Xcode.app - name: Build macOS run: | export JAVA_HOME=$JAVA_HOME_11_X64 # Skip freetype build on jdk11+ - if [ ${{ matrix.version }} != "jdk8u" ]; then + if [ ${{ matrix.version.name }} != "jdk8u" ]; then export BUILD_ARGS="--skip-freetype --make-exploded-image --create-sbom" ./build-farm/make-adopt-build-farm.sh export BUILD_ARGS="--assemble-exploded-image --create-sbom" @@ -151,7 +160,7 @@ jobs: ./build-farm/make-adopt-build-farm.sh fi env: - JAVA_TO_BUILD: ${{ matrix.version }} + JAVA_TO_BUILD: ${{ matrix.version.name }} ARCHITECTURE: x64 VARIANT: ${{ matrix.variant }} TARGET_OS: mac @@ -161,7 +170,7 @@ jobs: - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 name: Collect and Archive Artifacts with: - name: ${{matrix.version}}-${{matrix.os}}-${{matrix.variant}} + name: ${{matrix.version.name}}-${{matrix.os}}-${{matrix.variant}} path: workspace/target/* - name: Unpack jdk @@ -185,7 +194,7 @@ jobs: name: Collect and Archive SmokeTest Results if: failure() with: - name: "${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}_test_output" + name: "${{matrix.version.name}}-${{matrix.os}}-${{matrix.variant}}_test_output" path: ./**/output_*/ build_windows: diff --git a/build-farm/platform-specific-configurations/mac.sh b/build-farm/platform-specific-configurations/mac.sh index f3fff066a..e99a06366 100755 --- a/build-farm/platform-specific-configurations/mac.sh +++ b/build-farm/platform-specific-configurations/mac.sh @@ -47,11 +47,12 @@ then export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" fi else - if [[ "$JAVA_FEATURE_VERSION" -ge 17 ]] || [[ "${ARCHITECTURE}" == "aarch64" ]]; then + if [[ "$JAVA_FEATURE_VERSION" -ge 11 ]]; then # JDK17 requires metal (included in full xcode) as does JDK11 on aarch64 + # JDK11 on x64 is matched for consistency XCODE_SWITCH_PATH="/Applications/Xcode.app" else - # Command line tools used from JDK9-JDK16 + # Command line tools used from JDK9-JDK10 XCODE_SWITCH_PATH="/"; fi export PATH="/Users/jenkins/ccache-3.2.4:$PATH" From c7f1ad11a3c7a6d37622be2a689503142e473309 Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:15:21 +0000 Subject: [PATCH 03/15] riscv64: Use JDK21 to bootstrap JDK21 (#3539) Signed-off-by: Stewart X Addison --- build-farm/make-adopt-build-farm.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-farm/make-adopt-build-farm.sh b/build-farm/make-adopt-build-farm.sh index 4d8f2e1a4..bd406bf23 100755 --- a/build-farm/make-adopt-build-farm.sh +++ b/build-farm/make-adopt-build-farm.sh @@ -147,6 +147,10 @@ then # To support reproducible-builds the jar/jmod --date option is required # which is only available in jdk-17 and from jdk-19 so we cannot bootstrap with JDK16 JDK_BOOT_VERSION="17" + elif [ "${JAVA_FEATURE_VERSION}" == "21" ] && [ "${ARCHITECTURE}" == "riscv64" ]; then + # JDK20 has issues. No RVV fix for C910/C920 systems and + # does not run well in in docker containers + JDK_BOOT_VERSION="21" elif [ "${JAVA_FEATURE_VERSION}" == "19" ]; then JDK_BOOT_VERSION="19" fi From 8e2b810aa05db26577c199d9642cdb3b6986d8c1 Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:44:43 +0000 Subject: [PATCH 04/15] sbom: Updates to include builder name and CI link (#3537) * sbom: Updates to include builder name and CI link Signed-off-by: Stewart X Addison * Allow specific fallback entry if node/pipeline variable not available (e.g. not in jenkins Signed-off-by: Stewart X Addison --------- Signed-off-by: Stewart X Addison --- sbin/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbin/build.sh b/sbin/build.sh index 64dc87e38..feaa995f6 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -863,6 +863,10 @@ generateSBoM() { addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "OpenJDK Source Commit" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/openjdkSource.txt" # Add buildRef as JDK Component Property addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Temurin Build Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/buildSource.txt" + # Add jenkins job ID as JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Builder Job Reference" "${BUILD_URL:-N.A}" + # Add jenkins builder (agent/machine name) as JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Builder Name" "${NODE_NAME:-N.A}" # Add build timestamp addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Build Timestamp" "${BUILD_CONFIG[BUILD_TIMESTAMP]}" From 1efe1e1b30cb8f217b4a919ffc5f4668832211a1 Mon Sep 17 00:00:00 2001 From: sophia-guo Date: Wed, 29 Nov 2023 10:48:08 -0500 Subject: [PATCH 05/15] Update parsing build parameters and dependent tools (#3546) Signed-off-by: Sophia Guo --- tooling/linux_repro_build_compare.sh | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/tooling/linux_repro_build_compare.sh b/tooling/linux_repro_build_compare.sh index d6ef0db79..17c1b9bcb 100755 --- a/tooling/linux_repro_build_compare.sh +++ b/tooling/linux_repro_build_compare.sh @@ -27,7 +27,7 @@ installPrereqs() { if test -r /etc/redhat-release; then yum install -y gcc gcc-c++ make autoconf unzip zip alsa-lib-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXrandr-devel libXi-devel yum install -y file fontconfig fontconfig-devel systemtap-sdt-devel # Not included above ... - yum install -y git bzip2 xz openssl pigz which # pigz/which not strictly needed but help in final compression + yum install -y git bzip2 xz openssl pigz which jq # pigz/which not strictly needed but help in final compression if grep -i release.6 /etc/redhat-release; then if [ ! -r /usr/local/bin/autoconf ]; then curl https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz | tar xpfz - || exit 1 @@ -51,20 +51,6 @@ downloadAnt() { fi } -# get the TEMURIN_VERSION form the SBOM metadata -getTemurinVersion() { - TEMVER_MAJOR=$(grep '"major":' "$SBOM" | tr -d ' ,' | cut -d':' -f2) - TEMVER_MINOR=$(grep '"minor":' "$SBOM" | tr -d ' ,' | cut -d':' -f2) - TEMVER_SECURITY=$(grep '"security":' "$SBOM" | tr -d ' ,' | cut -d':' -f2) - TEMVER_BUILD=$(grep '"build":' "$SBOM" | tr -d ' ,' | cut -d':' -f2) - - TEMURIN_VERSION="$TEMVER_MAJOR" - if [ "$TEMVER_SECURITY" != "0" ]; then - TEMURIN_VERSION="$TEMURIN_VERSION.$TEMVER_MINOR.$TEMVER_SECURITY" - fi - TEMURIN_VERSION="$TEMURIN_VERSION+$TEMVER_BUILD" -} - setEnvironment() { export CC="${LOCALGCCDIR}/bin/gcc-${GCCVERSION}" export CXX="${LOCALGCCDIR}/bin/g++-${GCCVERSION}" @@ -105,13 +91,12 @@ downloadAnt echo "Retrieving and parsing SBOM from $SBOM_URL" curl -LO "$SBOM_URL" SBOM=$(basename "$SBOM_URL") -BOOTJDK_VERSION=$(grep configure_arguments "$SBOM" | tr ' ' \\n | grep ^Temurin- | tail -1 | cut -d- -f2) -GCCVERSION=$(tr ' ' \\n < "$SBOM" | grep CC= | cut -d- -f2 | cut -d\\ -f1) +BOOTJDK_VERSION=$(jq -r '.metadata.tools[] | select(.name == "BOOTJDK") | .version' "$SBOM") +GCCVERSION=$(jq -r '.metadata.tools[] | select(.name == "GCC") | .version' "$SBOM" | sed 's/.0$//') LOCALGCCDIR=/usr/local/gcc$(echo "$GCCVERSION" | cut -d. -f1) -TEMURIN_BUILD_SHA=$(awk -F'"' '/buildRef/{print$4}' "$SBOM" | cut -d/ -f7) -TEMURIN_BUILD_ARGS=$(grep makejdk_any_platform_args "$SBOM" | cut -d\" -f4 | sed -e "s/--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt/'--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt'/" -e "s/ --disable-warnings-as-errors --enable-dtrace/ '--disable-warnings-as-errors --enable-dtrace'/" -e 's/\\n//g' -e "s,--jdk-boot-dir [^ ]*,--jdk-boot-dir /usr/lib/jvm/jdk-$BOOTJDK_VERSION,g") - -getTemurinVersion +TEMURIN_BUILD_SHA=$(jq -r '.components[] | .properties[] | select (.name == "Temurin Build Ref") | .value' "$SBOM" | awk -F/ '{print $NF}') +TEMURIN_BUILD_ARGS=$(jq -r '.components[] | .properties[] | select (.name == "makejdk_any_platform_args") | .value' "$SBOM" | cut -d\" -f4 | sed -e "s/--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt/'--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt'/" -e "s/ --disable-warnings-as-errors --enable-dtrace/ '--disable-warnings-as-errors --enable-dtrace'/" -e 's/\\n//g' -e "s,--jdk-boot-dir [^ ]*,--jdk-boot-dir /usr/lib/jvm/jdk-$BOOTJDK_VERSION,g") +TEMURIN_VERSION=$(jq -r '.metadata.component.version' "$SBOM" | sed 's/-beta//' | cut -f1 -d"-") NATIVE_API_ARCH=$(uname -m) if [ "${NATIVE_API_ARCH}" = "x86_64" ]; then NATIVE_API_ARCH=x64; fi From 429cfe574d091ab5dbbfbdecd1daff7dd26064d2 Mon Sep 17 00:00:00 2001 From: Scott Fryer <60462088+steelhead31@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:53:47 +0000 Subject: [PATCH 06/15] Add required library for Configure to work on Ubuntu 22 onwards. (#3548) * Add library path for ubuntu 22 onwards for x64 * Tidy Ubuntu check logic * Linter fix * Linter fix --- build-farm/platform-specific-configurations/linux.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh index 94d43424e..22e7ec49d 100755 --- a/build-farm/platform-specific-configurations/linux.sh +++ b/build-farm/platform-specific-configurations/linux.sh @@ -194,6 +194,15 @@ function downloadBootJDK() if [ "${ARCHITECTURE}" == "x64" ] then export PATH=/opt/rh/devtoolset-2/root/usr/bin:$PATH + ## Fix For Issue https://github.com/adoptium/temurin-build/issues/3547 + ## Add Missing Library Path For Ubuntu 22+ + if [ -e /etc/os-release ]; then + ID=$(grep "^ID=" /etc/os-release | awk -F'=' '{print $2}') + INT_VERSION_ID=$(grep "^VERSION_ID=" /etc/os-release | awk -F'"' '{print $2}' | awk -F'.' '{print $1}') + if [ "$ID" == "ubuntu" ] && [ "$INT_VERSION_ID" -ge "22" ]; then + export LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH + fi + fi fi if [ "${ARCHITECTURE}" == "s390x" ] From 21af64b75fd7c311f839da6f06ae14bf30dc4c49 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Wed, 29 Nov 2023 19:28:48 +0000 Subject: [PATCH 07/15] Creating an automated build triage script (#3549) This will create a weekly issue that contains a list of the latest failing Temurin builds at Adoptium. Signed-off-by: Adam Farley --- .github/workflows/build-autotriage.yml | 32 ++ .../build_autotriage/autotriage_regexes.sh | 29 ++ tooling/build_autotriage/build_autotriage.sh | 279 ++++++++++++++++++ 3 files changed, 340 insertions(+) create mode 100644 .github/workflows/build-autotriage.yml create mode 100644 tooling/build_autotriage/autotriage_regexes.sh create mode 100644 tooling/build_autotriage/build_autotriage.sh diff --git a/.github/workflows/build-autotriage.yml b/.github/workflows/build-autotriage.yml new file mode 100644 index 000000000..e010b3391 --- /dev/null +++ b/.github/workflows/build-autotriage.yml @@ -0,0 +1,32 @@ +--- +# Runs a script to triage the latest timer-initiated Temurin build pipelines. + +name: "Build Autotriage" + +on: + schedule: + - cron: '0 0 * * MON' + push: + paths: + - '**build-autotriage.yml' + - '**build_autotriage.sh' + - '**autotriage_regexes.sh' + +env: + TRIAGE_SCRIPT: "${PWD}/tooling/build_autotriage/build_autotriage.sh" + +jobs: + Label: + runs-on: ubuntu-latest + name: Run Build Triage + steps: + - uses: actions/checkout@v3 + - name: "Run Build Auto Triage" + run: bash "${TRIAGE_SCRIPT}" jdk8u jdk11u jdk17u jdk21u jdk22head + + - name: Create Issue From File + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: JasonEtco/create-an-issue@v2 + with: + filename: ./build_triage_output.md \ No newline at end of file diff --git a/tooling/build_autotriage/autotriage_regexes.sh b/tooling/build_autotriage/autotriage_regexes.sh new file mode 100644 index 000000000..9a2a5999c --- /dev/null +++ b/tooling/build_autotriage/autotriage_regexes.sh @@ -0,0 +1,29 @@ +#!/bin/bash +################################################################################ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +################################################################################ +# +# This script produces arrays of regular expressions that match a type of failure +# that can be searched for in the output of an Eclipse Temurin build of OpenJDK. +# +# Each regular expression comes paired with metadata (providing useful information) +# +################################################################################ + +# declare -a arrayOfRegexes +# declare -a arrayOfRegexMetadata +# declare -a arrayOfRegexPreventability + +# TODO. \ No newline at end of file diff --git a/tooling/build_autotriage/build_autotriage.sh b/tooling/build_autotriage/build_autotriage.sh new file mode 100644 index 000000000..8ab8d2e74 --- /dev/null +++ b/tooling/build_autotriage/build_autotriage.sh @@ -0,0 +1,279 @@ +#!/bin/bash +# shellcheck disable=SC1091 +################################################################################ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +################################################################################ +# +# This script turns a list of Temurin build jobs into two things: +# 1. A markdown summary table that gives pass and fail numbers. +# 2. A list of each failing job/subjob link, plus information that can +# help identify the specific issue causing the failure. +# +################################################################################ + +declare -a arrayOfFailedJobs +# declare -a arrayOfFailedJobRegexs +declare -a arrayOfAllJDKVersions +declare -a arrayOfUs +declare -a buildIssues + +headJDKVersion=9999 + +# Imports arrayOfRegexes, arrayOfRegexMetadata, and arrayOfRegexPreventability +. ./tooling/build_autotriage/autotriage_regexes.sh + +# All temurin-available platforms. +declare -a temurinPlatforms +# The first jdk major version on that platform. +declare -a platformStart +# The last jdk major version on that platform ("99" for ongoing). +declare -a platformEnd + +temurinPlatforms+=("aix-ppc64"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("alpine-linux-aarch64"); platformStart+=(21); platformEnd+=(99) +temurinPlatforms+=("alpine-linux-x64"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("linux-aarch64"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("linux-arm"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("linux-ppc64le"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("linux-s390x"); platformStart+=(11); platformEnd+=(99) +temurinPlatforms+=("linux-x64"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("mac-aarch64"); platformStart+=(11); platformEnd+=(99) +temurinPlatforms+=("mac-x64"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("solaris-sparcv9"); platformStart+=(8); platformEnd+=(8) +temurinPlatforms+=("solaris-x64"); platformStart+=(8); platformEnd+=(8) +temurinPlatforms+=("windows-x64"); platformStart+=(8); platformEnd+=(99) +temurinPlatforms+=("windows-x86-32"); platformStart+=(8); platformEnd+=(17) + +errorLog() { + buildIssues+=("$1") + echo "ERROR FOUND: Issue ${#buildIssues[@]}: $1" +} + +# Parses the arguments to this script. +argumentParser() { + if [ "$#" -lt 1 ]; then + echo "Illegal argument/s. You must specify at least one jdk version." + exit 1 + fi + + while [[ $# -gt 0 ]] + do + if [[ ! $1 =~ ^jdk[0-9]+[u]?$ ]]; then + if [[ ! $1 =~ ^jdk[0-9]+head?$ ]]; then + echo "Script has failed. Reason: Illegal argument/s (name)." + echo "Correct argument format is: jdk#[u|head]" + exit 1 + fi + fi + + if [[ $1 =~ ^jdk[0-9]+$ ]]; then + arrayOfAllJDKVersions+=("${1:3}") + arrayOfUs+=("") + elif [[ $1 =~ ^jdk[0-9]+head$ ]]; then + arrayOfAllJDKVersions+=("${1:3:-4}") + arrayOfUs+=("") + headJDKVersion="${1:3:-4}" + else + arrayOfAllJDKVersions+=("${1:3:-1}") + arrayOfUs+=("u") + fi + + echo "JDK version identified: ${1:3}" + shift + done +} + +# Iterates over the supplied JDK versions and identifies the latest timer-triggered build URLs for each version. +# This function then checks that we're building Eclipse Temurin on every platform we should be, and makes a list +# of all the failing builds. +identifyFailedBuildsInTimerPipelines() { + # Iterate over jdk versions. + echo "Iterating over jdk versions." + for v in "${!arrayOfAllJDKVersions[@]}" + do + # First we find the latest timer-initiated pipeline for this JDK version. + echo "wgetting https://trss.adoptium.net/api/getBuildHistory?buildName=openjdk${arrayOfAllJDKVersions[v]}-pipeline" + latestTimerPipelineRaw=$(wget -q -O - "https://trss.adoptium.net/api/getBuildHistory?buildName=openjdk${arrayOfAllJDKVersions[v]}-pipeline") + latestTimerPipelineRaw="${latestTimerPipelineRaw},HereIsTheEndOfAVeryLongFile" + latestTimerPipeline="" + latestTimerJenkinsJobID="" + oldIFS=$IFS + IFS="," + for jsonEntry in $latestTimerPipelineRaw + do + if [[ $jsonEntry =~ ^\[\{\"_id\".* ]]; then + latestTimerPipeline=${jsonEntry:9:-1} + elif [[ $jsonEntry =~ ^\{\"_id\".* ]]; then + latestTimerPipeline=${jsonEntry:8:-1} + fi + + if [[ $jsonEntry =~ ^\"buildNum\"\:[0-9]+$ ]]; then + latestTimerJenkinsJobID=${jsonEntry:11} + fi + + if [[ ! $jsonEntry =~ .*user.* ]]; then + if [[ $jsonEntry =~ ^\"startBy\"\:\"timer\"[\}]?$ ]]; then + break + elif [[ $jsonEntry =~ ^\"startBy\"\:\".*build-scripts/weekly-openjdk.* ]]; then + break + elif [[ $jsonEntry =~ ^\"startBy\"\:\".*releaseTrigger_[0-9]+ea.* ]]; then + break + fi + fi + + if [[ $jsonEntry =~ ^HereIsTheEndOfAVeryLongFile$ ]]; then + errorLog "Could not find any timer/ea-tag triggered pipeline jobs for jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}. Skipping to the next jdk version." + continue 2 + fi + done + + echo "Found TRSS pipeline id for jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]} - ${latestTimerPipeline}" + echo "Whose URL is: https://ci.adoptium.net/job/build-scripts/job/openjdk${arrayOfAllJDKVersions[v]}-pipeline/${latestTimerJenkinsJobID}/" + + # Now grab a full list of builds launched by this pipeline. + jdkJenkinsJobVersion="jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}" + if [[ ${arrayOfAllJDKVersions[v]} -eq headJDKVersion ]]; then + jdkJenkinsJobVersion="jdk" + fi + echo "wgetting https://trss.adoptium.net/api/getAllChildBuilds?parentId=${latestTimerPipeline}&buildNameRegex=^jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}.*temurin$" + listOfPipelineBuilds=$(wget -q -O - "https://trss.adoptium.net/api/getAllChildBuilds?parentId=${latestTimerPipeline}&buildNameRegex=^${jdkJenkinsJobVersion}\-.*temurin$") + declare -a listOfBuildNames + declare -a listOfBuildNums + declare -a listOfBuildResults + + shorterListOfBuilds="" + for jsonEntry in $listOfPipelineBuilds + do + if [[ $jsonEntry =~ ^\"buildName\"\:.* ]]; then + listOfBuildNames+=("${jsonEntry:13:-1}") + shorterListOfBuilds+="${jsonEntry}," + elif [[ $jsonEntry =~ .*\"buildNum\"\.* ]]; then + listOfBuildNums+=("${jsonEntry:11}") + elif [[ $jsonEntry =~ .*\"buildResult\".* ]]; then + listOfBuildResults+=("${jsonEntry:15:-1}") + continue + fi + done + + echo "That pipeline's builds have been identified. Now validating them." + + IFS=$oldIFS + + # Now iterate over platforms to make sure we're launching every platform we should. + triageThesePlatforms="," + for p in "${!temurinPlatforms[@]}" + do + if [[ $shorterListOfBuilds =~ .*\"buildName\"\:\"${jdkJenkinsJobVersion}\-${temurinPlatforms[p]}\-temurin\".* ]]; then + if [[ ${arrayOfAllJDKVersions[v]} -lt ${platformStart[p]} ]]; then + errorLog "Error: Platform ${temurinPlatforms[p]} should not be built for ${jdkJenkinsJobVersion}. Will not triage." + continue + fi + if [[ ${arrayOfAllJDKVersions[v]} -gt ${platformEnd[p]} ]]; then + errorLog "Error: Platform ${temurinPlatforms[p]} should not be built for ${jdkJenkinsJobVersion}. Will not triage." + continue + fi + else + if [[ ${arrayOfAllJDKVersions[v]} -ge ${platformStart[p]} ]]; then + if [[ ${arrayOfAllJDKVersions[v]} -le ${platformEnd[p]} ]]; then + errorLog "Error: Platform ${temurinPlatforms[p]} should be built for ${jdkJenkinsJobVersion}, but was not launched." + echo "DEBUG: Looked for this: \"buildName\":\"${jdkJenkinsJobVersion}-${temurinPlatforms[p]}-temurin\"" + echo "DEBUG: In this: $shorterListOfBuilds" + echo "------" + continue + fi + fi + fi + # If we get to this stage of the loop, then this is a platform that was both *meant* to be built, and *was* built (or attempted). + triageThesePlatforms+="${jdkJenkinsJobVersion}-${temurinPlatforms[p]}-temurin," + done + + if [[ ${#triageThesePlatforms[@]} -gt 1 ]]; then + errorLog "Cannot find any valid build platforms launched by jdk ${arrayOfAllJDKVersions[v]}${arrayOfUs[v]} pipeline ${latestTimerJenkinsJobID}. Skipping to the next jdk version." + continue + fi + echo "Platforms validated. Identifying build numbers for these platforms: ${triageThesePlatforms:1:-1}" + + for b in "${!listOfBuildNames[@]}" + do + if [[ $triageThesePlatforms =~ .*,${listOfBuildNames[$b]},.* ]]; then + if [[ ! ${listOfBuildResults[$b]} =~ ^SUCCESS$ ]]; then + if [[ ! ${listOfBuildResults[$b]} =~ ^UNSTABLE$ ]]; then + jdkJenkinsJobVersion="jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}" + if [[ ${arrayOfAllJDKVersions[v]} -eq headJDKVersion ]]; then + jdkJenkinsJobVersion="jdk" + fi + failedJobLink="https://ci.adoptium.net/job/build-scripts/job/jobs/job/${jdkJenkinsJobVersion}/job/${listOfBuildNames[$b]}/${listOfBuildNums[$b]}/" + echo "Identified a failed build for triage: ${failedJobLink}" + arrayOfFailedJobs+=("${failedJobLink}") + fi + fi + fi + done + echo "Build numbers found, and failures will be added to the array of builds to be triaged." + done +} + +# Takes a single failed jenkins build job URL as a string, and identifies the source of +# the failure if possible. +buildFailureTriager() { + echo "Attempting to triage a job: ${1}" + echo "- Failed job: ${1}" >> build_triage_output.md + # Todo: Iterate over the failures found and triage them against the pending array of regexes. + # For now we'll put them in a tidy md-style file for issue inclusion. + +} + +startOutputFile() { + { echo "---"; + echo "name: Build Issue Summary"; + echo "about: For triaging the nightly and weekend build failures"; + echo "title: Build Issue Summary for {{ date | date('YYYY-MM-DD') }}"; + echo "labels: 'weekly-build-triage'"; + echo "---"; + echo ""; + } >> build_triage_output.md +} + +# @@@@@@@@@@@@ Script execution starts here @@@@@@@@@@@@ + +echo "Build AutoTriage is starting." + +argumentParser "$@" + +identifyFailedBuildsInTimerPipelines + +startOutputFile + +if [[ ${#arrayOfFailedJobs[@]} -gt 0 ]]; then + echo "# Failed Builds" >> build_triage_output.md + for failedJob in "${arrayOfFailedJobs[@]}" + do + buildFailureTriager "$failedJob" + done + echo "# End of list" >> build_triage_output.md +else + echo "All build jobs passed. Huzzah!" +fi + +if [[ ${#buildIssues[@]} -gt 0 ]]; then + echo "# Script Issues" >> build_triage_output.md + for issueID in "${!buildIssues[@]}" + do + echo "- Issue ${issueID}: ${buildIssues[issueID]}\n" >> build_triage_output.md + done + echo "# End of Issues" >> build_triage_output.md +fi + +echo "Build AutoTriage is complete." From 0d421dac151e56395d0f8021324129595a0292aa Mon Sep 17 00:00:00 2001 From: Scott Fryer <60462088+steelhead31@users.noreply.github.com> Date: Thu, 30 Nov 2023 12:25:26 +0000 Subject: [PATCH 08/15] Add required library for Configure to work on Ubuntu 22 onwards. ( for both Arm64 & x64 ) (#3551) * Increase scope of ubuntu22 fix * Make LIBRARY_PATH generic * Linter fix --- .../platform-specific-configurations/linux.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh index 22e7ec49d..3fea89ac3 100755 --- a/build-farm/platform-specific-configurations/linux.sh +++ b/build-farm/platform-specific-configurations/linux.sh @@ -194,14 +194,16 @@ function downloadBootJDK() if [ "${ARCHITECTURE}" == "x64" ] then export PATH=/opt/rh/devtoolset-2/root/usr/bin:$PATH - ## Fix For Issue https://github.com/adoptium/temurin-build/issues/3547 - ## Add Missing Library Path For Ubuntu 22+ - if [ -e /etc/os-release ]; then - ID=$(grep "^ID=" /etc/os-release | awk -F'=' '{print $2}') - INT_VERSION_ID=$(grep "^VERSION_ID=" /etc/os-release | awk -F'"' '{print $2}' | awk -F'.' '{print $1}') - if [ "$ID" == "ubuntu" ] && [ "$INT_VERSION_ID" -ge "22" ]; then - export LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH - fi +fi + +## Fix For Issue https://github.com/adoptium/temurin-build/issues/3547 +## Add Missing Library Path For Ubuntu 22+ +if [ -e /etc/os-release ]; then + ID=$(grep "^ID=" /etc/os-release | awk -F'=' '{print $2}') + INT_VERSION_ID=$(grep "^VERSION_ID=" /etc/os-release | awk -F'"' '{print $2}' | awk -F'.' '{print $1}') + LIB_ARCH=$(uname -m)-linux-gnu + if [ "$ID" == "ubuntu" ] && [ "$INT_VERSION_ID" -ge "22" ]; then + export LIBRARY_PATH=/usr/lib/$LIB_ARCH:$LIBRARY_PATH fi fi From 26601733e8bc6faa50f9dc6ee39958ab7f7021f1 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Thu, 30 Nov 2023 15:19:02 +0000 Subject: [PATCH 09/15] Make autotriage only run weekly (#3556) Make autotriage only run weekly This should prevent accidental triggering when someone fetches a commit containing a change to these files and pushes it to their fork. Also moving a pwd envvar as its current location may cause problems in some forks. Signed-off-by: Adam Farley --- .github/workflows/build-autotriage.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-autotriage.yml b/.github/workflows/build-autotriage.yml index e010b3391..296807426 100644 --- a/.github/workflows/build-autotriage.yml +++ b/.github/workflows/build-autotriage.yml @@ -6,14 +6,9 @@ name: "Build Autotriage" on: schedule: - cron: '0 0 * * MON' - push: - paths: - - '**build-autotriage.yml' - - '**build_autotriage.sh' - - '**autotriage_regexes.sh' - + env: - TRIAGE_SCRIPT: "${PWD}/tooling/build_autotriage/build_autotriage.sh" + TRIAGE_SCRIPT: "tooling/build_autotriage/build_autotriage.sh" jobs: Label: @@ -22,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: "Run Build Auto Triage" - run: bash "${TRIAGE_SCRIPT}" jdk8u jdk11u jdk17u jdk21u jdk22head + run: bash "${PWD}/${TRIAGE_SCRIPT}" jdk8u jdk11u jdk17u jdk21u jdk22head - name: Create Issue From File env: From d23434ce3031dcaecce0b1c9eeccb90fafd3e3b0 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Thu, 30 Nov 2023 17:40:13 +0100 Subject: [PATCH 10/15] Add workaround for brew update failure when running aqa smoke tests (#3543) * Add workaround for brew update failure * Update .github/workflows/build.yml Co-authored-by: Martijn Verburg --------- Co-authored-by: Martijn Verburg --- .github/workflows/build.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 887f28531..35c0dc9b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,6 +123,22 @@ jobs: steps: - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + # https://github.com/actions/runner-images/issues/6817 + - name: (Mac) Workaround for homebrew + shell: bash + if: runner.os == 'macOS' + run: | + rm /usr/local/bin/2to3 || true + rm /usr/local/bin/2to3-3.11 || true + rm /usr/local/bin/idle3 || true + rm /usr/local/bin/idle3.11 || true + rm /usr/local/bin/pydoc3 || true + rm /usr/local/bin/pydoc3.11 || true + rm /usr/local/bin/python3 || true + rm /usr/local/bin/python3.11 || true + rm /usr/local/bin/python3-config || true + rm /usr/local/bin/python3.11-config || true + - name: Install Dependencies run: | brew install automake bash binutils freetype gnu-sed nasm From da2408e4ea988090835f15f29cb170873cced045 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Dec 2023 15:32:10 +0000 Subject: [PATCH 11/15] WIP: Rerun gpg recv-keys command multiple times (#3544) Rerun the recv-keys command 10 times if it fails This allows us to hopefully provide some tolerance against the timeout errors, while at the same time using the same keyserver; fixing the problem while not modifying our risk profile. Signed-off-by: Adam Farley --- sbin/prepareWorkspace.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh index 5f44dee93..caddef32d 100644 --- a/sbin/prepareWorkspace.sh +++ b/sbin/prepareWorkspace.sh @@ -336,7 +336,18 @@ checkingAndDownloadingAlsa() { # Note: the uptime command below is to aid diagnostics for this issue: # https://github.com/adoptium/temurin-build/issues/3518#issuecomment-1792606345 uptime - gpg --keyserver keyserver.ubuntu.com --keyserver-options timeout=300 --recv-keys "${ALSA_LIB_GPGKEYID}" + # Will retry command below until it passes or we've failed 10 times. + for i in {1..10}; do + if gpg --keyserver keyserver.ubuntu.com --keyserver-options timeout=300 --recv-keys "${ALSA_LIB_GPGKEYID}"; then + echo "gpg command has passed." + break + elif [[ ${i} -lt 10 ]]; then + echo "gpg recv-keys attempt has failed. Retrying after 10 second pause..." + sleep 10s + else + echo "ERROR: gpg recv-keys final attempt has failed. Will not try again." + fi + done echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key "${ALSA_LIB_GPGKEYID}" trust; gpg --verify alsa-lib.tar.bz2.sig alsa-lib.tar.bz2 || exit 1 From 25f518f216aca92aa5ac76c9184e4c0bbbbeb9ae Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Tue, 5 Dec 2023 14:19:39 +0000 Subject: [PATCH 12/15] Update CycloneDX core jar to 8.0.3 for 1.5 spec (#3558) Signed-off-by: Stewart X Addison --- cyclonedx-lib/build.xml | 2 +- cyclonedx-lib/getDependencies | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cyclonedx-lib/build.xml b/cyclonedx-lib/build.xml index 04e47b20d..f2bc74648 100644 --- a/cyclonedx-lib/build.xml +++ b/cyclonedx-lib/build.xml @@ -43,7 +43,7 @@ diff --git a/cyclonedx-lib/getDependencies b/cyclonedx-lib/getDependencies index aaeedf81d..897d3b980 100644 --- a/cyclonedx-lib/getDependencies +++ b/cyclonedx-lib/getDependencies @@ -22,8 +22,11 @@ def fetchDeps() { timeout(time: time_limit, unit: 'HOURS') { try { sh 'mkdir sbom_dependencies' - - def cyclonedx_core_java_version = "7.3.2" + + + # These versions come from https://github.com/CycloneDX/cyclonedx-core-java/tags + # Version->spec mappings are in https://github.com/CycloneDX/cyclonedx-core-java#cyclonedx-schema-support + def cyclonedx_core_java_version = "8.0.3" def jackson_core_version = "2.14.2" def jackson_annotations_version = "2.14.2" def jackson_databind_version = "2.14.2" @@ -32,7 +35,7 @@ def fetchDeps() { def commons_io_version = "2.11.0" def github_package_url_version = "1.4.1" - fetchSingleFile("cyclonedx-core-java.jar", "88193228f85a955127dc73e1c72efc9e08e18a01d227df47d0865dc20eceffd1", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar") + fetchSingleFile("cyclonedx-core-java.jar", "ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar") fetchSingleFile("jackson-core.jar", "b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a", "com/fasterxml/jackson/core/jackson-core/${jackson_core_version}/jackson-core-${jackson_core_version}.jar") fetchSingleFile("jackson-dataformat-xml.jar", "edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b", "com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson_databind_version}/jackson-dataformat-xml-${jackson_databind_version}.jar") fetchSingleFile("jackson-databind.jar", "501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424", "com/fasterxml/jackson/core/jackson-databind/${jackson_databind_version}/jackson-databind-${jackson_databind_version}.jar") From 3a3f4acda4cb8904eddbee9b96432a2fe6c59dcc Mon Sep 17 00:00:00 2001 From: Andrew Leonard <31470007+andrew-m-leonard@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:11:42 +0000 Subject: [PATCH 13/15] Default Temurin jdk-21+ to use bundled FreeType (#3557) * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default Temurin jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard * Default jdk-21+ to use bundled FreeType Signed-off-by: Andrew Leonard --------- Signed-off-by: Andrew Leonard --- .github/workflows/build.yml | 4 ++-- build-farm/platform-specific-configurations/aix.sh | 9 +++++++-- .../platform-specific-configurations/alpine-linux.sh | 9 +++++++-- build-farm/platform-specific-configurations/linux.sh | 10 ++++++++-- build-farm/platform-specific-configurations/mac.sh | 5 +++++ build-farm/platform-specific-configurations/windows.sh | 5 +++++ .../src/net/adoptium/test/BundledFreetypeTest.java | 6 +++++- 7 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35c0dc9b9..d7763c3e3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -165,9 +165,9 @@ jobs: - name: Build macOS run: | export JAVA_HOME=$JAVA_HOME_11_X64 - # Skip freetype build on jdk11+ + # jdk11u+ uses two part exploded & assemble build if [ ${{ matrix.version.name }} != "jdk8u" ]; then - export BUILD_ARGS="--skip-freetype --make-exploded-image --create-sbom" + export BUILD_ARGS="--make-exploded-image --create-sbom" ./build-farm/make-adopt-build-farm.sh export BUILD_ARGS="--assemble-exploded-image --create-sbom" ./build-farm/make-adopt-build-farm.sh diff --git a/build-farm/platform-specific-configurations/aix.sh b/build-farm/platform-specific-configurations/aix.sh index 2babd73a5..fea9acefb 100755 --- a/build-farm/platform-specific-configurations/aix.sh +++ b/build-farm/platform-specific-configurations/aix.sh @@ -59,12 +59,17 @@ then export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-extra-ldflags=-lpthread --with-extra-cflags=-lpthread --with-extra-cxxflags=-lpthread" fi -export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" +if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then + # jdk-21+ uses "bundled" FreeType + export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled" +else + export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" +fi if [ "${VARIANT}" == "${BUILD_VARIANT_OPENJ9}" ]; then export LDR_CNTRL=MAXDATA=0x80000000 fi -echo LDR_CNTRL=$LDR_CNTRL +echo LDR_CNTRL="$LDR_CNTRL" BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR" if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then diff --git a/build-farm/platform-specific-configurations/alpine-linux.sh b/build-farm/platform-specific-configurations/alpine-linux.sh index 5aee36e90..1b3eedbcf 100644 --- a/build-farm/platform-specific-configurations/alpine-linux.sh +++ b/build-farm/platform-specific-configurations/alpine-linux.sh @@ -28,8 +28,13 @@ fi # ccache seems flaky on alpine export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --disable-ccache" -# We don't bundle freetype on alpine anymore, and expect the user to have it. -export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" +if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then + # jdk-21+ uses "bundled" FreeType + export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled" +else + # We don't bundle freetype on alpine anymore, and expect the user to have it. + export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" +fi BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR" if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh index 3fea89ac3..655cd16c0 100755 --- a/build-farm/platform-specific-configurations/linux.sh +++ b/build-farm/platform-specific-configurations/linux.sh @@ -18,8 +18,14 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=sbin/common/constants.sh source "$SCRIPT_DIR/../../sbin/common/constants.sh" -# Bundling our own freetype can cause problems, so we skip that on linux. -export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" + +if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then + # jdk-21+ uses "bundled" FreeType + export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled" +else + # Bundling our own freetype can cause problems, so we skip that on linux. + export BUILD_ARGS="${BUILD_ARGS} --skip-freetype" +fi NATIVE_API_ARCH=$(uname -m) if [ "${NATIVE_API_ARCH}" = "x86_64" ]; then NATIVE_API_ARCH=x64; fi diff --git a/build-farm/platform-specific-configurations/mac.sh b/build-farm/platform-specific-configurations/mac.sh index e99a06366..7283ceb82 100755 --- a/build-farm/platform-specific-configurations/mac.sh +++ b/build-farm/platform-specific-configurations/mac.sh @@ -68,6 +68,11 @@ else fi fi +if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then + # jdk-21+ uses "bundled" FreeType + export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled" +fi + # The configure option '--with-macosx-codesign-identity' is supported in JDK8 OpenJ9 and JDK11 and JDK14+ if [[ ( "$JAVA_FEATURE_VERSION" -eq 11 ) || ( "$JAVA_FEATURE_VERSION" -ge 14 ) ]] then diff --git a/build-farm/platform-specific-configurations/windows.sh b/build-farm/platform-specific-configurations/windows.sh index e41514035..25d5756a8 100755 --- a/build-farm/platform-specific-configurations/windows.sh +++ b/build-farm/platform-specific-configurations/windows.sh @@ -231,6 +231,11 @@ then fi fi +if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then + # jdk-21+ uses "bundled" FreeType + export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled" +fi + if [ "${ARCHITECTURE}" == "aarch64" ]; then export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --disable-ccache --openjdk-target=aarch64-unknown-cygwin" fi diff --git a/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java b/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java index baf413120..a6e103201 100644 --- a/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java +++ b/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java @@ -74,7 +74,11 @@ public void freetypeOnlyBundledOnCertainPlatforms() throws IOException { .filter(name -> freetypePattern.matcher(name).matches()) .collect(Collectors.toSet()); - if (jdkPlatform.runsOn(OperatingSystem.MACOS)) { + if (jdkVersion.isNewerOrEqual(21)) { + // jdk-21+ uses "bundled" FreeType + assertTrue(freetypeFiles.size() > 0, + "Expected libfreetype.dylib to be bundled but it is not."); + } else if (jdkPlatform.runsOn(OperatingSystem.MACOS)) { assertTrue(freetypeFiles.size() > 0, "Expected libfreetype.dylib to be bundled but it is not."); } else if (jdkPlatform.runsOn(OperatingSystem.WINDOWS)) { From feccf849eeabba055f2c1842a3e4f9bf697f04be Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:20:43 +0000 Subject: [PATCH 14/15] Fixup comment characters in getDependencies (#3564) Signed-off-by: Stewart X Addison --- cyclonedx-lib/getDependencies | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cyclonedx-lib/getDependencies b/cyclonedx-lib/getDependencies index 897d3b980..fbbdca11b 100644 --- a/cyclonedx-lib/getDependencies +++ b/cyclonedx-lib/getDependencies @@ -24,8 +24,8 @@ def fetchDeps() { sh 'mkdir sbom_dependencies' - # These versions come from https://github.com/CycloneDX/cyclonedx-core-java/tags - # Version->spec mappings are in https://github.com/CycloneDX/cyclonedx-core-java#cyclonedx-schema-support + // These versions come from https://github.com/CycloneDX/cyclonedx-core-java/tags + // Version->spec mappings are in https://github.com/CycloneDX/cyclonedx-core-java#cyclonedx-schema-support def cyclonedx_core_java_version = "8.0.3" def jackson_core_version = "2.14.2" def jackson_annotations_version = "2.14.2" From 5c41fcb579e7bfa70c0a8b48617858d82ebd700b Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Tue, 5 Dec 2023 17:23:53 +0100 Subject: [PATCH 15/15] Add subject info the sbom (#3529) * Add generated components to the sbom, add a hash of each component. * Store fullVer as PRODUCT_HOME is not available anymore after the archives have been created. * Cleanup. * Rebase * Address linter comments. * Fix joinPath when leading slashes are in a part. * Determine the sbom target file name more robust. * Use sha function from prepareWorkspace.sh. * Add a joinPathOS method that converts paths to OS specific ones and use plan joinPath for files that are passed to cygwin programs. * Add fail-safe method to get the target file name for any component. * Make linter happy. --- .../src/temurin/sbom/TemurinGenSBOM.java | 97 ++++--- sbin/build.sh | 251 +++++++++++------- sbin/common/common.sh | 15 ++ sbin/common/sbom.sh | 13 +- 4 files changed, 246 insertions(+), 130 deletions(-) diff --git a/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java b/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java index 9d1a4308b..fce3789c1 100644 --- a/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java +++ b/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java @@ -41,18 +41,18 @@ public final class TemurinGenSBOM { private TemurinGenSBOM() { } + /** * Main entry. * @param args Arguments for sbom operation. */ - public static void main(final String[] args) { String cmd = null; String comment = null; String compName = null; String description = null; String fileName = null; - String hashes = null; + String hash = null; String name = null; String tool = null; String type = null; @@ -73,8 +73,8 @@ public static void main(final String[] args) { url = args[++i]; } else if (args[i].equals("--comment")) { comment = args[++i]; - } else if (args[i].equals("--hashes")) { - hashes = args[++i]; + } else if (args[i].equals("--hash")) { + hash = args[++i]; } else if (args[i].equals("--compName")) { compName = args[++i]; } else if (args[i].equals("--description")) { @@ -85,15 +85,17 @@ public static void main(final String[] args) { tool = args[++i]; } else if (args[i].equals("--createNewSBOM")) { cmd = "createNewSBOM"; - } else if (args[i].equals("--addMetadata")) { // Metadata Component. We can set "name" for Metadata. + } else if (args[i].equals("--addMetadata")) { // Metadata Component. We can set "name" for Metadata. cmd = "addMetadata"; - } else if (args[i].equals("--addMetadataComponent")) { // Metadata Component. We can set "name" for Metadata->Component. + } else if (args[i].equals("--addMetadataComponent")) { // Metadata Component. We can set "name" for Metadata->Component. cmd = "addMetadataComponent"; - } else if (args[i].equals("--addMetadataProp")) { // MetaData Component --> Property -> name-value + } else if (args[i].equals("--addMetadataProp")) { // MetaData Component --> Property -> name-value cmd = "addMetadataProperty"; - } else if (args[i].equals("--addComponent")) { // Components->Property: will add name-value. + } else if (args[i].equals("--addComponent")) { cmd = "addComponent"; - } else if (args[i].equals("--addComponentProp")) { // Components->Property: will add name-value. + } else if (args[i].equals("--addComponentHash")) { + cmd = "addComponentHash"; + } else if (args[i].equals("--addComponentProp")) { // Components --> Property: will add name-value. cmd = "addComponentProp"; } else if (args[i].equals("--addExternalReference")) { cmd = "addExternalReference"; @@ -106,22 +108,22 @@ public static void main(final String[] args) { } } switch (cmd) { - case "createNewSBOM": // Creates JSON file + case "createNewSBOM": // Creates JSON file Bom bom = createBom(); writeJSONfile(bom, fileName); break; - case "addMetadata": // Adds Metadata --> name + case "addMetadata": // Adds Metadata --> name bom = addMetadata(fileName); writeJSONfile(bom, fileName); break; - case "addMetadataComponent": // Adds Metadata --> Component--> name + case "addMetadataComponent": // Adds Metadata --> Component --> name bom = addMetadataComponent(fileName, name, type, version, description); writeJSONfile(bom, fileName); break; - case "addMetadataProperty": // Adds MetaData--> Property --> name-value: + case "addMetadataProperty": // Adds MetaData --> Property --> name-value: bom = addMetadataProperty(fileName, name, value); writeJSONfile(bom, fileName); break; @@ -131,23 +133,28 @@ public static void main(final String[] args) { writeJSONfile(bom, fileName); break; - case "addComponent": // Adds Component + case "addComponent": // Adds Components --> Component --> name bom = addComponent(fileName, compName, version, description); writeJSONfile(bom, fileName); break; - case "addComponentProp": // Adds Components --> name-value pairs + case "addComponentHash": // Adds Components --> Component --> hash + bom = addComponentHash(fileName, compName, hash); + writeJSONfile(bom, fileName); + break; + + case "addComponentProp": // Adds Components --> Component --> name-value pairs bom = addComponentProperty(fileName, compName, name, value); writeJSONfile(bom, fileName); break; - case "addExternalReference": // Adds external Reference - bom = addExternalReference(fileName, hashes, url, comment); + case "addExternalReference": // Adds external Reference + bom = addExternalReference(fileName, hash, url, comment); writeJSONfile(bom, fileName); break; - case "addComponentExternalReference": // Adds external Reference to component - bom = addComponentExternalReference(fileName, hashes, url, comment); + case "addComponentExternalReference": // Adds external Reference to component + bom = addComponentExternalReference(fileName, hash, url, comment); writeJSONfile(bom, fileName); break; default: @@ -163,7 +170,9 @@ static Bom createBom() { Bom bom = new Bom(); return bom; } - static Bom addMetadata(final String fileName) { // Method to store metadata --> name + + // Method to store Metadata --> name. + static Bom addMetadata(final String fileName) { Bom bom = readJSONfile(fileName); Metadata meta = new Metadata(); OrganizationalEntity org = new OrganizationalEntity(); @@ -176,6 +185,7 @@ static Bom addMetadata(final String fileName) { // Method to store meta bom.setMetadata(meta); return bom; } + static Bom addMetadataComponent(final String fileName, final String name, final String type, final String version, final String description) { Bom bom = readJSONfile(fileName); Metadata meta = new Metadata(); @@ -196,7 +206,9 @@ static Bom addMetadataComponent(final String fileName, final String name, final bom.setMetadata(meta); return bom; } - static Bom addMetadataProperty(final String fileName, final String name, final String value) { // Method to store metadata --> Properties List --> name-values + + // Method to store Metadata --> Properties List --> name-values. + static Bom addMetadataProperty(final String fileName, final String name, final String value) { Bom bom = readJSONfile(fileName); Metadata meta = new Metadata(); Property prop1 = new Property(); @@ -207,6 +219,7 @@ static Bom addMetadataProperty(final String fileName, final String name, final S bom.setMetadata(meta); return bom; } + static Bom addMetadataTools(final String fileName, final String toolName, final String version) { Bom bom = readJSONfile(fileName); Metadata meta = new Metadata(); @@ -218,7 +231,9 @@ static Bom addMetadataTools(final String fileName, final String toolName, final bom.setMetadata(meta); return bom; } - static Bom addComponent(final String fileName, final String compName, final String version, final String description) { // Method to store Component --> name & single name-value pair + + // Method to store Component --> name & single name-value pair. + static Bom addComponent(final String fileName, final String compName, final String version, final String description) { Bom bom = readJSONfile(fileName); Component comp = new Component(); comp.setName(compName); @@ -226,12 +241,26 @@ static Bom addComponent(final String fileName, final String compName, final Stri comp.setType(Component.Type.FRAMEWORK); comp.setDescription(description); comp.setGroup("adoptium.net"); - comp.setAuthor("Adoptium Temurin"); + comp.setAuthor("Eclipse Temurin"); comp.setPublisher("Eclipse Temurin"); bom.addComponent(comp); return bom; } - static Bom addComponentProperty(final String fileName, final String compName, final String name, final String value) { // Method to add Component --> Property --> name-value pairs + + static Bom addComponentHash(final String fileName, final String compName, final String hash) { + Bom bom = readJSONfile(fileName); + List componentArrayList = bom.getComponents(); + for (Component item : componentArrayList) { + if (item.getName().equals(compName)) { + Hash hash1 = new Hash(Hash.Algorithm.SHA_256, hash); + item.addHash(hash1); + } + } + return bom; + } + + // Method to add Component --> Property --> name-value pairs. + static Bom addComponentProperty(final String fileName, final String compName, final String name, final String value) { Bom bom = readJSONfile(fileName); List componentArrayList = bom.getComponents(); for (Component item : componentArrayList) { @@ -244,21 +273,25 @@ static Bom addComponentProperty(final String fileName, final String compName, fi } return bom; } - static Bom addExternalReference(final String fileName, final String hashes, final String url, final String comment) { // Method to store externalReferences: dependency_version_alsa + + // Method to store externalReferences: dependency_version_alsa. + static Bom addExternalReference(final String fileName, final String hash, final String url, final String comment) { Bom bom = readJSONfile(fileName); ExternalReference extRef = new ExternalReference(); - Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hashes); + Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hash); extRef.setType(ExternalReference.Type.BUILD_SYSTEM); //required - extRef.setUrl(url); // required must be a valid URL with protocal + extRef.setUrl(url); // required must be a valid URL with protocol extRef.setComment(comment); extRef.addHash(hash1); bom.addExternalReference(extRef); return bom; } - static Bom addComponentExternalReference(final String fileName, final String hashes, final String url, final String comment) { // Method to store externalReferences to store: openjdk_source + + // Method to store externalReferences to store: openjdk_source. + static Bom addComponentExternalReference(final String fileName, final String hash, final String url, final String comment) { Bom bom = readJSONfile(fileName); ExternalReference extRef = new ExternalReference(); - Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hashes); + Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hash); Component comp = new Component(); extRef.addHash(hash1); extRef.setUrl(url); @@ -276,7 +309,8 @@ static String generateBomJson(final Bom bom) { return json; } - static void writeJSONfile(final Bom bom, final String fileName) { // Creates testJson.json file + // Writes the BOM object to the specified file. + static void writeJSONfile(final Bom bom, final String fileName) { FileWriter file; String json = generateBomJson(bom); try { @@ -288,7 +322,8 @@ static void writeJSONfile(final Bom bom, final String fileName) { // Cr } } - static Bom readJSONfile(final String fileName) { // Returns parse bom + // Returns a parsed BOM object from the specified file. + static Bom readJSONfile(final String fileName) { Bom bom = null; try { FileReader reader = new FileReader(fileName); diff --git a/sbin/build.sh b/sbin/build.sh index feaa995f6..faae6d926 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -669,7 +669,7 @@ createSourceArchive() { # shellcheck disable=SC2001 srcArchiveName="$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]}" | sed 's/_x64_linux_hotspot_/-sources_/g')" else - srcArchiveName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-sources}") + srcArchiveName=$(getTargetFileNameForComponent "sources") fi local oldPwd="${PWD}" @@ -791,6 +791,11 @@ setupAntEnv() { echo "Unable to find a suitable JAVA_HOME to build the cyclonedx-lib" exit 2 fi + + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then + javaHome=$(cygpath -w "${javaHome}") + fi + echo "${javaHome}" } @@ -808,14 +813,12 @@ buildCyclonedxLib() { JAVA_HOME=${javaHome} ant -f "${ANTBUILDFILE}" build } -# Generate the SBoM -generateSBoM() { - local javaHome="${1}" +# get the classpath to run the CycloneDX java app TemurinGenSBOM +getCyclonedxClasspath() { - # classpath to run CycloneDX java app TemurinGenSBOM - CYCLONEDB_JAR_DIR="${CYCLONEDB_DIR}/build/jar" - classpath="${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar:${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar:${CYCLONEDB_JAR_DIR}/jackson-core.jar:${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar:${CYCLONEDB_JAR_DIR}/jackson-databind.jar:${CYCLONEDB_JAR_DIR}/jackson-annotations.jar:${CYCLONEDB_JAR_DIR}/json-schema.jar:${CYCLONEDB_JAR_DIR}/commons-codec.jar:${CYCLONEDB_JAR_DIR}/commons-io.jar:${CYCLONEDB_JAR_DIR}/github-package-url.jar" - sbomJson="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/sbom.json" + local CYCLONEDB_JAR_DIR="${CYCLONEDB_DIR}/build/jar" + + local classpath="${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar:${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar:${CYCLONEDB_JAR_DIR}/jackson-core.jar:${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar:${CYCLONEDB_JAR_DIR}/jackson-databind.jar:${CYCLONEDB_JAR_DIR}/jackson-annotations.jar:${CYCLONEDB_JAR_DIR}/json-schema.jar:${CYCLONEDB_JAR_DIR}/commons-codec.jar:${CYCLONEDB_JAR_DIR}/commons-io.jar:${CYCLONEDB_JAR_DIR}/github-package-url.jar" if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then classpath="" for jarfile in "${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar" "${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar" \ @@ -826,17 +829,40 @@ generateSBoM() { do classpath+=$(cygpath -w "${jarfile}")";" done - sbomJson=$(cygpath -w "${sbomJson}") - javaHome=$(cygpath -w "${javaHome}") fi + echo "${classpath}" +} + +# Generate the SBoM +generateSBoM() { + if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "false" ]] || [[ ! -d "${CYCLONEDB_DIR}" ]]; then + echo "Skip generating SBOM" + return + fi + + local javaHome="$(setupAntEnv)" + + buildCyclonedxLib "${javaHome}" + # classpath to run java app TemurinGenSBOM + local classpath="$(getCyclonedxClasspath)" + + local sbomTargetName=$(getTargetFileNameForComponent "sbom") + # Remove the tarball / zip extension from the name to be used for the SBOM + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then + sbomTargetName=$(echo "${sbomTargetName}.json" | sed "s/\.zip//") + else + sbomTargetName=$(echo "${sbomTargetName}.json" | sed "s/\.tar\.gz//") + fi + + local sbomJson="$(joinPathOS ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} ${sbomTargetName})" + echo "OpenJDK SBOM will be ${sbomJson}." + # Clean any old json rm -f "${sbomJson}" - # Run a series of SBOM API commands to generate the required SBOM - JAVA_LOC="$PRODUCT_HOME/bin/java" - local fullVer=$($JAVA_LOC -XshowSettings:properties -version 2>&1 | grep 'java.runtime.version' | sed 's/^.*= //' | tr -d '\r') - local fullVerOutput=$($JAVA_LOC -version 2>&1) + local fullVer=$(cat "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersion.txt") + local fullVerOutput=$(cat "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersionOutput.txt") # Create initial SBOM json createSBOMFile "${javaHome}" "${classpath}" "${sbomJson}" @@ -844,46 +870,13 @@ generateSBoM() { addSBOMMetadata "${javaHome}" "${classpath}" "${sbomJson}" # Create component to metadata in SBOM - addSBOMMetadataComponent "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "framework" "${fullVer}" "Temurin JDK Component" + addSBOMMetadataComponent "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "framework" "${fullVer}" "Eclipse Temurin components" # Below add property to metadata # Add OS full version (Kernel is covered in the first field) addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "OS version" "${BUILD_CONFIG[OS_FULL_VERSION]^}" addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "OS architecture" "${BUILD_CONFIG[OS_ARCHITECTURE]^}" - # Create JDK Component - addSBOMComponent "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "${fullVer}" "${BUILD_CONFIG[BUILD_VARIANT]^} JDK Component" - - # Below add different properties to JDK component - # Add variant as JDK Component Property - addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "JDK Variant" "${BUILD_CONFIG[BUILD_VARIANT]^}" - # Add scmRef as JDK Component Property - addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "SCM Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/scmref.txt" - # Add OpenJDK source ref commit as JDK Component Property - addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "OpenJDK Source Commit" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/openjdkSource.txt" - # Add buildRef as JDK Component Property - addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Temurin Build Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/buildSource.txt" - # Add jenkins job ID as JDK Component Property - addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Builder Job Reference" "${BUILD_URL:-N.A}" - # Add jenkins builder (agent/machine name) as JDK Component Property - addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Builder Name" "${NODE_NAME:-N.A}" - - # Add build timestamp - addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Build Timestamp" "${BUILD_CONFIG[BUILD_TIMESTAMP]}" - - # Add Tool Summary section from configure.txt - checkingToolSummary - addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Build Tools Summary" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/dependency_tool_sum.txt" - # Add builtConfig JDK Component Property, load as Json string - built_config=$(createConfigToJsonString) - addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "Build Config" "${built_config}" - # Add full_version_output JDK Component Property - addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "full_version_output" "${fullVerOutput}" - # Add makejdk_any_platform_args JDK Component Property - addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "makejdk_any_platform_args" "${BUILD_CONFIG[WORKSPACE_DIR]}/config/makejdk-any-platform.args" - # Add make_command_args JDK Component Property - addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "make_command_args" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/makeCommandArg.txt" - # Below add build tools into metadata tools if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "linux" ]; then addGLIBCforLinux @@ -897,10 +890,13 @@ generateSBoM() { # Add FreeType 3rd party addFreeTypeVersionInfo # Add FreeMarker 3rd party (openj9) - addSBOMMetadataTools "${javaHome}" "${classpath}" "${sbomJson}" "FreeMarker" "$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/dependency_version_freemarker.txt)" - + local freemarker_version="$(joinPathOS ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} 'metadata/dependency_version_freemarker.txt')" + if [ -f "${freemarker_version}" ]; then + addSBOMMetadataTools "${javaHome}" "${classpath}" "${sbomJson}" "FreeMarker" "$(cat ${freemarker_version})" + fi + # Add Build Docker image SHA1 - buildimagesha=$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/docker.txt) + local buildimagesha=$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/docker.txt) # ${BUILD_CONFIG[USE_DOCKER]^} always set to false cannot rely on it. if [ -n "${buildimagesha}" ] && [ "${buildimagesha}" != "N.A" ]; then addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "Use Docker for build" "true" @@ -909,13 +905,83 @@ generateSBoM() { addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "Use Docker for build" "false" fi + checkingToolSummary + + # add individual components that have been generated in this build + local components=("JDK" "JRE" "SOURCES" "STATIC-LIBS" "DEBUGIMAGE" "TESTIMAGE") + for component in "${components[@]}" + do + local componentLowerCase=$(echo "${component}" | tr '[:upper:]' '[:lower:]') + + local componentName="${component} Component" + # shellcheck disable=SC2001 + local archiveName=$(getTargetFileNameForComponent "${componentLowerCase}") + local archiveFile="$(joinPath ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} ${archiveName})" + + # special handling for static-libs, determine the glibc type that is used. + if [ "${component}" == "STATIC-LIBS" ]; then + local staticLibsVariants=("" "-glibc" "-musl") + for staticLibsVariant in "${staticLibsVariants[@]}" + do + # shellcheck disable=SC2001 + archiveName=$(getTargetFileNameForComponent "static-libs${staticLibsVariant}") + archiveFile="$(joinPath ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} ${archiveName})" + if [ -f "${archiveFile}" ]; then + break + fi + done + fi + + if [ ! -f "${archiveFile}" ]; then + continue + fi + + local sha=$(sha256File "${archiveFile}") + + # Create JDK Component + addSBOMComponent "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "${fullVer}" "${BUILD_CONFIG[BUILD_VARIANT]^} ${component} Component" + + # Add SHA256 hash for the component + addSBOMComponentHash "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "${sha}" + + # Below add different properties to JDK component + # Add target archive name as JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Filename" "${archiveName}" + # Add variant as JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "JDK Variant" "${BUILD_CONFIG[BUILD_VARIANT]^}" + # Add scmRef as JDK Component Property + addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "SCM Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/scmref.txt" + # Add OpenJDK source ref commit as JDK Component Property + addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "OpenJDK Source Commit" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/openjdkSource.txt" + # Add buildRef as JDK Component Property + addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Temurin Build Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/buildSource.txt" + # Add jenkins job ID as JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Builder Job Reference" "${BUILD_URL:-N.A}" + # Add jenkins builder (agent/machine name) as JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Builder Name" "${NODE_NAME:-N.A}" + + # Add build timestamp + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Build Timestamp" "${BUILD_CONFIG[BUILD_TIMESTAMP]}" + + # Add Tool Summary section from configure.txt + addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Build Tools Summary" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/dependency_tool_sum.txt" + # Add builtConfig JDK Component Property, load as Json string + built_config=$(createConfigToJsonString) + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Build Config" "${built_config}" + # Add full_version_output JDK Component Property + addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "full_version_output" "${fullVerOutput}" + # Add makejdk_any_platform_args JDK Component Property + addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "makejdk_any_platform_args" "${BUILD_CONFIG[WORKSPACE_DIR]}/config/makejdk-any-platform.args" + # Add make_command_args JDK Component Property + addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "make_command_args" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/makeCommandArg.txt" + done + # Print SBOM json echo "CycloneDX SBOM:" cat "${sbomJson}" echo "" } - # Generate build tools info into dependency file checkingToolSummary() { echo "Checking and getting Tool Summary info:" @@ -1114,7 +1180,7 @@ printJavaVersionString() { exit 3 elif [ "${BUILD_CONFIG[CROSSCOMPILE]}" == "true" ]; then # job is cross compiled, so we cannot run it on the build system - # So we leave it for now and retrive the version from a downstream job after the build + # So we leave it for now and retrieve the version from a downstream job after the build echo "Warning: java version can't be run on cross compiled build system. Faking version for now..." else # print version string around easy to find output @@ -1178,7 +1244,6 @@ cleanAndMoveArchiveFiles() { local testImageTargetPath=$(getTestImageArchivePath) local debugImageTargetPath=$(getDebugImageArchivePath) local staticLibsImageTargetPath=$(getStaticLibsArchivePath) - local sbomTargetPath=$(getSbomArchivePath) echo "Moving archive content to target archive paths and cleaning unnecessary files..." @@ -1216,15 +1281,6 @@ cleanAndMoveArchiveFiles() { mv "${testImagePath}" "${testImageTargetPath}" fi - # If creating SBOM, move it to the target Sbom archive path - if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "true" ]]; then - local sbomJson="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/sbom.json" - echo "moving ${sbomJson} to ${sbomTargetPath}/sbom.json" - rm -rf "${sbomTargetPath}" || true - mkdir "${sbomTargetPath}" - mv "${sbomJson}" "${sbomTargetPath}" - fi - # Static libs image - check if the directory exists local staticLibsImagePath="${BUILD_CONFIG[STATIC_LIBS_IMAGE_PATH]}" local osArch @@ -1686,6 +1742,21 @@ createArchive() { fi } +# Gets the target file name for a given component in a fail-safe way. +getTargetFileNameForComponent() { + local component=$1 + local target_file_name=${BUILD_CONFIG[TARGET_FILE_NAME]} + + # check if the target file name contains a -jdk pattern to be replaced with the component. + if [[ "${target_file_name}" == *"-jdk"* ]]; then + # shellcheck disable=SC2001 + echo "${target_file_name}" | sed "s/-jdk/-${component}/" + else + # if no pattern is found, append the component name right before the extension. + echo "${target_file_name}" | sed -r "s/(.+)(\.tar\.gz|\.zip)/\1-${component}\2/" + fi +} + # Create a Tar ball createOpenJDKTarArchive() { local jdkTargetPath=$(getJdkArchivePath) @@ -1693,13 +1764,12 @@ createOpenJDKTarArchive() { local testImageTargetPath=$(getTestImageArchivePath) local debugImageTargetPath=$(getDebugImageArchivePath) local staticLibsImageTargetPath=$(getStaticLibsArchivePath) - local sbomTargetPath=$(getSbomArchivePath) echo "OpenJDK JDK path will be ${jdkTargetPath}. JRE path will be ${jreTargetPath}" if [ -d "${jreTargetPath}" ]; then # shellcheck disable=SC2001 - local jreName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]}" | sed 's/-jdk/-jre/') + local jreName=$(getTargetFileNameForComponent "jre") # for macOS system, code sign directory before creating tar.gz file if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "darwin" ] && [ -n "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" ]; then codesign --options runtime --timestamp --sign "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" "${jreTargetPath}" @@ -1707,18 +1777,18 @@ createOpenJDKTarArchive() { createArchive "${jreTargetPath}" "${jreName}" fi if [ -d "${testImageTargetPath}" ]; then - echo "OpenJDK test image path will be ${testImageTargetPath}." - local testImageName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-testimage}") + local testImageName=$(getTargetFileNameForComponent "testimage") + echo "OpenJDK test image archive file name will be ${testImageName}." createArchive "${testImageTargetPath}" "${testImageName}" fi if [ -d "${debugImageTargetPath}" ]; then - echo "OpenJDK debug image path will be ${debugImageTargetPath}." - local debugImageName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-debugimage}") + local debugImageName=$(getTargetFileNameForComponent "debugimage") + echo "OpenJDK debug image archive file name will be ${debugImageName}." createArchive "${debugImageTargetPath}" "${debugImageName}" fi if [ -d "${staticLibsImageTargetPath}" ]; then echo "OpenJDK static libs path will be ${staticLibsImageTargetPath}." - local staticLibsTag="-static-libs" + local staticLibsTag="static-libs" if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" = "linux" ]; then # on Linux there might be glibc and musl variants of this local cLib="glibc" @@ -1730,25 +1800,10 @@ createOpenJDKTarArchive() { staticLibsTag="${staticLibsTag}-${cLib}" fi # shellcheck disable=SC2001 - local staticLibsImageName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]}" | sed "s/-jdk/${staticLibsTag}/g") + local staticLibsImageName=$(getTargetFileNameForComponent "${staticLibsTag}") echo "OpenJDK static libs archive file name will be ${staticLibsImageName}." createArchive "${staticLibsImageTargetPath}" "${staticLibsImageName}" fi - if [ -d "${sbomTargetPath}" ]; then - # SBOM archive artifact as a .json file - local sbomTargetName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-sbom}.json") - - # Remove the tarball extension from the name to be used for the SBOM - if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then - sbomTargetName="${sbomTargetName//\.zip/}" - else - sbomTargetName="${sbomTargetName//\.tar\.gz/}" - fi - - local sbomArchiveTarget=${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/${sbomTargetName} - echo "OpenJDK SBOM will be ${sbomTargetName}." - cp "${sbomTargetPath}/sbom.json" "${sbomArchiveTarget}" - fi # for macOS system, code sign directory before creating tar.gz file if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "darwin" ] && [ -n "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" ]; then codesign --options runtime --timestamp --sign "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" "${jdkTargetPath}" @@ -1980,6 +2035,7 @@ addInfoToJson(){ addVendorToJson addSourceToJson # Build repository and commit SHA addOpenJDKSourceToJson # OpenJDK repository and commit SHA + addProductVersionToJson } addVariantVersionToJson(){ @@ -2022,6 +2078,15 @@ addOpenJDKSourceToJson() { # name of git repo for which SOURCE sha is from fi } +addProductVersionToJson() { + local JAVA_LOC="$PRODUCT_HOME/bin/java" + local fullVer=$(${JAVA_LOC} -XshowSettings:properties -version 2>&1 | grep 'java.runtime.version' | sed 's/^.*= //' | tr -d '\r') + local fullVerOutput=$(${JAVA_LOC} -version 2>&1) + + echo "${fullVer}" > "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersion.txt" 2>&1 + echo "${fullVerOutput}" > "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersionOutput.txt" 2>&1 +} + ################################################################################ loadConfigFromFile @@ -2037,17 +2102,12 @@ if [[ "${BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]}" == "true" ]]; then printJavaVersionString addInfoToReleaseFile addInfoToJson - if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "true" ]] && [[ -d "${CYCLONEDB_DIR}" ]]; then - javaHome="$(setupAntEnv)" - buildCyclonedxLib "${javaHome}" - generateSBoM "${javaHome}" - unset javaHome - fi cleanAndMoveArchiveFiles copyFreeFontForMacOS setPlistForMacOS addNoticeFile createOpenJDKTarArchive + generateSBoM exit 0 fi @@ -2073,17 +2133,12 @@ if [[ "${BUILD_CONFIG[MAKE_EXPLODED]}" != "true" ]]; then printJavaVersionString addInfoToReleaseFile addInfoToJson - if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "true" ]] && [[ -d "${CYCLONEDB_DIR}" ]]; then - javaHome="$(setupAntEnv)" - buildCyclonedxLib "${javaHome}" - generateSBoM "${javaHome}" - unset javaHome - fi cleanAndMoveArchiveFiles copyFreeFontForMacOS setPlistForMacOS addNoticeFile createOpenJDKTarArchive + generateSBoM fi echo "build.sh : $(date +%T) : All done!" diff --git a/sbin/common/common.sh b/sbin/common/common.sh index 26b35418c..638da169e 100755 --- a/sbin/common/common.sh +++ b/sbin/common/common.sh @@ -96,6 +96,21 @@ function setDockerVolumeSuffix() { fi } +# Joins multiple parts to a valid file path for the current OS +function joinPathOS() { + local path=$(printf '/%s' "${@}" | sed 's|/\+|/|g') + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then + path=$(cygpath -w "${path}") + fi + echo "${path}" +} + +# Joins multiple parts to a valid file path using slashes +function joinPath() { + local path=$(printf '/%s' "${@}" | sed 's|/\+|/|g') + echo "${path}" +} + # Create a Tar ball getArchiveExtension() { diff --git a/sbin/common/sbom.sh b/sbin/common/sbom.sh index 1cc3d07bd..f7fe0caf9 100755 --- a/sbin/common/sbom.sh +++ b/sbin/common/sbom.sh @@ -23,7 +23,7 @@ verifySBOMSignature() { "${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinSignSBOM --verifySBOMSignature --jsonFile "${jsonFile}" --publicKeyFile "${publicKeyFile}" } -# Set basic SBMO metadata with timestamp, authors, manufacture to ${sbomJson} +# Set basic SBOM metadata with timestamp, authors, manufacture to ${sbomJson} addSBOMMetadata() { local javaHome="${1}" local classpath="${2}" @@ -120,6 +120,17 @@ addSBOMComponentFromFile() { "${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinGenSBOM --addComponentProp --jsonFile "${jsonFile}" --compName "${compName}" --name "${name}" --value "${value}" } +# Ref: https://cyclonedx.org/docs/1.4/json/#components_items_hashes +# Add the given sha256 hash to the given SBOM Component +addSBOMComponentHash() { + local javaHome="${1}" + local classpath="${2}" + local jsonFile="${3}" + local compName="${4}" + local hash="${5}" + "${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinGenSBOM --addComponentHash --jsonFile "${jsonFile}" --compName "${compName}" --hash "${hash}" +} + # Ref: https://cyclonedx.org/docs/1.4/json/#components_items_properties # Add the given Property name & value to the given SBOM Component addSBOMComponentProperty() {