diff --git a/.github/workflows/build-autotriage.yml b/.github/workflows/build-autotriage.yml
new file mode 100644
index 000000000..296807426
--- /dev/null
+++ b/.github/workflows/build-autotriage.yml
@@ -0,0 +1,27 @@
+---
+# Runs a script to triage the latest timer-initiated Temurin build pipelines.
+
+name: "Build Autotriage"
+
+on:
+ schedule:
+ - cron: '0 0 * * MON'
+
+env:
+ TRIAGE_SCRIPT: "tooling/build_autotriage/build_autotriage.sh"
+
+jobs:
+ Label:
+ runs-on: ubuntu-latest
+ name: Run Build Triage
+ steps:
+ - uses: actions/checkout@v3
+ - name: "Run Build Auto Triage"
+ run: bash "${PWD}/${TRIAGE_SCRIPT}" jdk8u jdk11u jdk17u jdk21u jdk22head
+
+ - name: Create Issue From File
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ uses: JasonEtco/create-an-issue@v2
+ with:
+ filename: ./build_triage_output.md
\ No newline at end of file
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 13cd50c9f..d7763c3e3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -109,17 +109,36 @@ jobs:
build_macos:
name: macOS
- runs-on: macos-11
+ runs-on: ${{ matrix.version.distro }}
strategy:
fail-fast: false
matrix:
os: [macOS]
- version: [jdk8u, jdk11u]
+ version: [
+ { name: jdk8u, distro: macos-11 },
+ { name: jdk11u, distro: macos-13 },
+ ]
variant: [temurin]
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+ # https://github.com/actions/runner-images/issues/6817
+ - name: (Mac) Workaround for homebrew
+ shell: bash
+ if: runner.os == 'macOS'
+ run: |
+ rm /usr/local/bin/2to3 || true
+ rm /usr/local/bin/2to3-3.11 || true
+ rm /usr/local/bin/idle3 || true
+ rm /usr/local/bin/idle3.11 || true
+ rm /usr/local/bin/pydoc3 || true
+ rm /usr/local/bin/pydoc3.11 || true
+ rm /usr/local/bin/python3 || true
+ rm /usr/local/bin/python3.11 || true
+ rm /usr/local/bin/python3-config || true
+ rm /usr/local/bin/python3.11-config || true
+
- name: Install Dependencies
run: |
brew install automake bash binutils freetype gnu-sed nasm
@@ -129,20 +148,26 @@ jobs:
with:
java-version: 7
distribution: 'zulu'
- if: matrix.version == 'jdk8u'
+ if: matrix.version.name == 'jdk8u'
- - name: Select correct Xcode
+ - name: Select correct Xcode (JDK8)
+ if: matrix.version.name == 'jdk8u'
run: |
rm -rf /Applications/Xcode.app
ln -s /Applications/Xcode_11.7.app /Applications/Xcode.app
- ln -s /Applications/Xcode_11.7.app /Applications/Xcode-11.7.app
+
+ - name: Select correct Xcode (JDK11+)
+ if: matrix.version.name != 'jdk8u'
+ run: |
+ rm -rf /Applications/Xcode.app
+ ln -s /Applications/Xcode_15.0.1.app /Applications/Xcode.app
- name: Build macOS
run: |
export JAVA_HOME=$JAVA_HOME_11_X64
- # Skip freetype build on jdk11+
- if [ ${{ matrix.version }} != "jdk8u" ]; then
- export BUILD_ARGS="--skip-freetype --make-exploded-image --create-sbom"
+ # jdk11u+ uses two part exploded & assemble build
+ if [ ${{ matrix.version.name }} != "jdk8u" ]; then
+ export BUILD_ARGS="--make-exploded-image --create-sbom"
./build-farm/make-adopt-build-farm.sh
export BUILD_ARGS="--assemble-exploded-image --create-sbom"
./build-farm/make-adopt-build-farm.sh
@@ -151,7 +176,7 @@ jobs:
./build-farm/make-adopt-build-farm.sh
fi
env:
- JAVA_TO_BUILD: ${{ matrix.version }}
+ JAVA_TO_BUILD: ${{ matrix.version.name }}
ARCHITECTURE: x64
VARIANT: ${{ matrix.variant }}
TARGET_OS: mac
@@ -161,7 +186,7 @@ jobs:
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
name: Collect and Archive Artifacts
with:
- name: ${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}
+ name: ${{matrix.version.name}}-${{matrix.os}}-${{matrix.variant}}
path: workspace/target/*
- name: Unpack jdk
@@ -185,7 +210,7 @@ jobs:
name: Collect and Archive SmokeTest Results
if: failure()
with:
- name: "${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}_test_output"
+ name: "${{matrix.version.name}}-${{matrix.os}}-${{matrix.variant}}_test_output"
path: ./**/output_*/
build_windows:
diff --git a/build-farm/make-adopt-build-farm.sh b/build-farm/make-adopt-build-farm.sh
index 4d8f2e1a4..bd406bf23 100755
--- a/build-farm/make-adopt-build-farm.sh
+++ b/build-farm/make-adopt-build-farm.sh
@@ -147,6 +147,10 @@ then
# To support reproducible-builds the jar/jmod --date option is required
# which is only available in jdk-17 and from jdk-19 so we cannot bootstrap with JDK16
JDK_BOOT_VERSION="17"
+ elif [ "${JAVA_FEATURE_VERSION}" == "21" ] && [ "${ARCHITECTURE}" == "riscv64" ]; then
+ # JDK20 has issues. No RVV fix for C910/C920 systems and
+ # does not run well in in docker containers
+ JDK_BOOT_VERSION="21"
elif [ "${JAVA_FEATURE_VERSION}" == "19" ]; then
JDK_BOOT_VERSION="19"
fi
diff --git a/build-farm/platform-specific-configurations/aix.sh b/build-farm/platform-specific-configurations/aix.sh
index 2babd73a5..fea9acefb 100755
--- a/build-farm/platform-specific-configurations/aix.sh
+++ b/build-farm/platform-specific-configurations/aix.sh
@@ -59,12 +59,17 @@ then
export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-extra-ldflags=-lpthread --with-extra-cflags=-lpthread --with-extra-cxxflags=-lpthread"
fi
-export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
+if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then
+ # jdk-21+ uses "bundled" FreeType
+ export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled"
+else
+ export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
+fi
if [ "${VARIANT}" == "${BUILD_VARIANT_OPENJ9}" ]; then
export LDR_CNTRL=MAXDATA=0x80000000
fi
-echo LDR_CNTRL=$LDR_CNTRL
+echo LDR_CNTRL="$LDR_CNTRL"
BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR"
if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then
diff --git a/build-farm/platform-specific-configurations/alpine-linux.sh b/build-farm/platform-specific-configurations/alpine-linux.sh
index 5aee36e90..1b3eedbcf 100644
--- a/build-farm/platform-specific-configurations/alpine-linux.sh
+++ b/build-farm/platform-specific-configurations/alpine-linux.sh
@@ -28,8 +28,13 @@ fi
# ccache seems flaky on alpine
export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --disable-ccache"
-# We don't bundle freetype on alpine anymore, and expect the user to have it.
-export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
+if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then
+ # jdk-21+ uses "bundled" FreeType
+ export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled"
+else
+ # We don't bundle freetype on alpine anymore, and expect the user to have it.
+ export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
+fi
BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR"
if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then
diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh
index 94d43424e..655cd16c0 100755
--- a/build-farm/platform-specific-configurations/linux.sh
+++ b/build-farm/platform-specific-configurations/linux.sh
@@ -18,8 +18,14 @@
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck source=sbin/common/constants.sh
source "$SCRIPT_DIR/../../sbin/common/constants.sh"
-# Bundling our own freetype can cause problems, so we skip that on linux.
-export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
+
+if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then
+ # jdk-21+ uses "bundled" FreeType
+ export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled"
+else
+ # Bundling our own freetype can cause problems, so we skip that on linux.
+ export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
+fi
NATIVE_API_ARCH=$(uname -m)
if [ "${NATIVE_API_ARCH}" = "x86_64" ]; then NATIVE_API_ARCH=x64; fi
@@ -196,6 +202,17 @@ then
export PATH=/opt/rh/devtoolset-2/root/usr/bin:$PATH
fi
+## Fix For Issue https://github.com/adoptium/temurin-build/issues/3547
+## Add Missing Library Path For Ubuntu 22+
+if [ -e /etc/os-release ]; then
+ ID=$(grep "^ID=" /etc/os-release | awk -F'=' '{print $2}')
+ INT_VERSION_ID=$(grep "^VERSION_ID=" /etc/os-release | awk -F'"' '{print $2}' | awk -F'.' '{print $1}')
+ LIB_ARCH=$(uname -m)-linux-gnu
+ if [ "$ID" == "ubuntu" ] && [ "$INT_VERSION_ID" -ge "22" ]; then
+ export LIBRARY_PATH=/usr/lib/$LIB_ARCH:$LIBRARY_PATH
+ fi
+fi
+
if [ "${ARCHITECTURE}" == "s390x" ]
then
export LANG=C
diff --git a/build-farm/platform-specific-configurations/mac.sh b/build-farm/platform-specific-configurations/mac.sh
index f3fff066a..7283ceb82 100755
--- a/build-farm/platform-specific-configurations/mac.sh
+++ b/build-farm/platform-specific-configurations/mac.sh
@@ -47,11 +47,12 @@ then
export BUILD_ARGS="${BUILD_ARGS} --skip-freetype"
fi
else
- if [[ "$JAVA_FEATURE_VERSION" -ge 17 ]] || [[ "${ARCHITECTURE}" == "aarch64" ]]; then
+ if [[ "$JAVA_FEATURE_VERSION" -ge 11 ]]; then
# JDK17 requires metal (included in full xcode) as does JDK11 on aarch64
+ # JDK11 on x64 is matched for consistency
XCODE_SWITCH_PATH="/Applications/Xcode.app"
else
- # Command line tools used from JDK9-JDK16
+ # Command line tools used from JDK9-JDK10
XCODE_SWITCH_PATH="/";
fi
export PATH="/Users/jenkins/ccache-3.2.4:$PATH"
@@ -67,6 +68,11 @@ else
fi
fi
+if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then
+ # jdk-21+ uses "bundled" FreeType
+ export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled"
+fi
+
# The configure option '--with-macosx-codesign-identity' is supported in JDK8 OpenJ9 and JDK11 and JDK14+
if [[ ( "$JAVA_FEATURE_VERSION" -eq 11 ) || ( "$JAVA_FEATURE_VERSION" -ge 14 ) ]]
then
diff --git a/build-farm/platform-specific-configurations/windows.sh b/build-farm/platform-specific-configurations/windows.sh
index e41514035..25d5756a8 100755
--- a/build-farm/platform-specific-configurations/windows.sh
+++ b/build-farm/platform-specific-configurations/windows.sh
@@ -231,6 +231,11 @@ then
fi
fi
+if [[ "$JAVA_FEATURE_VERSION" -ge 21 ]]; then
+ # jdk-21+ uses "bundled" FreeType
+ export BUILD_ARGS="${BUILD_ARGS} --freetype-dir bundled"
+fi
+
if [ "${ARCHITECTURE}" == "aarch64" ]; then
export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --disable-ccache --openjdk-target=aarch64-unknown-cygwin"
fi
diff --git a/cyclonedx-lib/build.xml b/cyclonedx-lib/build.xml
index 27682ffa0..e2f8f2e2f 100644
--- a/cyclonedx-lib/build.xml
+++ b/cyclonedx-lib/build.xml
@@ -43,7 +43,7 @@
diff --git a/cyclonedx-lib/getDependencies b/cyclonedx-lib/getDependencies
index aaeedf81d..fbbdca11b 100644
--- a/cyclonedx-lib/getDependencies
+++ b/cyclonedx-lib/getDependencies
@@ -22,8 +22,11 @@ def fetchDeps() {
timeout(time: time_limit, unit: 'HOURS') {
try {
sh 'mkdir sbom_dependencies'
-
- def cyclonedx_core_java_version = "7.3.2"
+
+
+ // These versions come from https://github.com/CycloneDX/cyclonedx-core-java/tags
+ // Version->spec mappings are in https://github.com/CycloneDX/cyclonedx-core-java#cyclonedx-schema-support
+ def cyclonedx_core_java_version = "8.0.3"
def jackson_core_version = "2.14.2"
def jackson_annotations_version = "2.14.2"
def jackson_databind_version = "2.14.2"
@@ -32,7 +35,7 @@ def fetchDeps() {
def commons_io_version = "2.11.0"
def github_package_url_version = "1.4.1"
- fetchSingleFile("cyclonedx-core-java.jar", "88193228f85a955127dc73e1c72efc9e08e18a01d227df47d0865dc20eceffd1", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar")
+ fetchSingleFile("cyclonedx-core-java.jar", "ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar")
fetchSingleFile("jackson-core.jar", "b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a", "com/fasterxml/jackson/core/jackson-core/${jackson_core_version}/jackson-core-${jackson_core_version}.jar")
fetchSingleFile("jackson-dataformat-xml.jar", "edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b", "com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson_databind_version}/jackson-dataformat-xml-${jackson_databind_version}.jar")
fetchSingleFile("jackson-databind.jar", "501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424", "com/fasterxml/jackson/core/jackson-databind/${jackson_databind_version}/jackson-databind-${jackson_databind_version}.jar")
diff --git a/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java b/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java
index 75c6608a5..b0e08a252 100644
--- a/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java
+++ b/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java
@@ -43,18 +43,18 @@ public final class TemurinGenSBOM {
private TemurinGenSBOM() {
}
+
/**
* Main entry.
* @param args Arguments for sbom operation.
*/
-
public static void main(final String[] args) {
String cmd = null;
String comment = null;
String compName = null;
String description = null;
String fileName = null;
- String hashes = null;
+ String hash = null;
String name = null;
String tool = null;
String type = null;
@@ -75,8 +75,8 @@ public static void main(final String[] args) {
url = args[++i];
} else if (args[i].equals("--comment")) {
comment = args[++i];
- } else if (args[i].equals("--hashes")) {
- hashes = args[++i];
+ } else if (args[i].equals("--hash")) {
+ hash = args[++i];
} else if (args[i].equals("--compName")) {
compName = args[++i];
} else if (args[i].equals("--description")) {
@@ -87,15 +87,17 @@ public static void main(final String[] args) {
tool = args[++i];
} else if (args[i].equals("--createNewSBOM")) {
cmd = "createNewSBOM";
- } else if (args[i].equals("--addMetadata")) { // Metadata Component. We can set "name" for Metadata.
+ } else if (args[i].equals("--addMetadata")) { // Metadata Component. We can set "name" for Metadata.
cmd = "addMetadata";
- } else if (args[i].equals("--addMetadataComponent")) { // Metadata Component. We can set "name" for Metadata->Component.
+ } else if (args[i].equals("--addMetadataComponent")) { // Metadata Component. We can set "name" for Metadata->Component.
cmd = "addMetadataComponent";
- } else if (args[i].equals("--addMetadataProp")) { // MetaData Component --> Property -> name-value
+ } else if (args[i].equals("--addMetadataProp")) { // MetaData Component --> Property -> name-value
cmd = "addMetadataProperty";
- } else if (args[i].equals("--addComponent")) { // Components->Property: will add name-value.
+ } else if (args[i].equals("--addComponent")) {
cmd = "addComponent";
- } else if (args[i].equals("--addComponentProp")) { // Components->Property: will add name-value.
+ } else if (args[i].equals("--addComponentHash")) {
+ cmd = "addComponentHash";
+ } else if (args[i].equals("--addComponentProp")) { // Components --> Property: will add name-value.
cmd = "addComponentProp";
} else if (args[i].equals("--addExternalReference")) {
cmd = "addExternalReference";
@@ -114,22 +116,22 @@ public static void main(final String[] args) {
}
}
switch (cmd) {
- case "createNewSBOM": // Creates JSON file
+ case "createNewSBOM": // Creates JSON file
Bom bom = createBom();
writeJSONfile(bom, fileName);
break;
- case "addMetadata": // Adds Metadata --> name
+ case "addMetadata": // Adds Metadata --> name
bom = addMetadata(fileName);
writeJSONfile(bom, fileName);
break;
- case "addMetadataComponent": // Adds Metadata --> Component--> name
+ case "addMetadataComponent": // Adds Metadata --> Component --> name
bom = addMetadataComponent(fileName, name, type, version, description);
writeJSONfile(bom, fileName);
break;
- case "addMetadataProperty": // Adds MetaData--> Property --> name-value:
+ case "addMetadataProperty": // Adds MetaData --> Property --> name-value:
bom = addMetadataProperty(fileName, name, value);
writeJSONfile(bom, fileName);
break;
@@ -153,23 +155,28 @@ public static void main(final String[] args) {
writeJSONfile(bom, fileName);
break;
- case "addComponent": // Adds Component
+ case "addComponent": // Adds Components --> Component --> name
bom = addComponent(fileName, compName, version, description);
writeJSONfile(bom, fileName);
break;
- case "addComponentProp": // Adds Components --> name-value pairs
+ case "addComponentHash": // Adds Components --> Component --> hash
+ bom = addComponentHash(fileName, compName, hash);
+ writeJSONfile(bom, fileName);
+ break;
+
+ case "addComponentProp": // Adds Components --> Component --> name-value pairs
bom = addComponentProperty(fileName, compName, name, value);
writeJSONfile(bom, fileName);
break;
- case "addExternalReference": // Adds external Reference
- bom = addExternalReference(fileName, hashes, url, comment);
+ case "addExternalReference": // Adds external Reference
+ bom = addExternalReference(fileName, hash, url, comment);
writeJSONfile(bom, fileName);
break;
- case "addComponentExternalReference": // Adds external Reference to component
- bom = addComponentExternalReference(fileName, hashes, url, comment);
+ case "addComponentExternalReference": // Adds external Reference to component
+ bom = addComponentExternalReference(fileName, hash, url, comment);
writeJSONfile(bom, fileName);
break;
default:
@@ -185,7 +192,9 @@ static Bom createBom() {
Bom bom = new Bom();
return bom;
}
- static Bom addMetadata(final String fileName) { // Method to store metadata --> name
+
+ // Method to store Metadata --> name.
+ static Bom addMetadata(final String fileName) {
Bom bom = readJSONfile(fileName);
Metadata meta = new Metadata();
OrganizationalEntity org = new OrganizationalEntity();
@@ -198,6 +207,7 @@ static Bom addMetadata(final String fileName) { // Method to store meta
bom.setMetadata(meta);
return bom;
}
+
static Bom addMetadataComponent(final String fileName, final String name, final String type, final String version, final String description) {
Bom bom = readJSONfile(fileName);
Metadata meta = new Metadata();
@@ -218,7 +228,9 @@ static Bom addMetadataComponent(final String fileName, final String name, final
bom.setMetadata(meta);
return bom;
}
- static Bom addMetadataProperty(final String fileName, final String name, final String value) { // Method to store metadata --> Properties List --> name-values
+
+ // Method to store Metadata --> Properties List --> name-values.
+ static Bom addMetadataProperty(final String fileName, final String name, final String value) {
Bom bom = readJSONfile(fileName);
Metadata meta = new Metadata();
Property prop1 = new Property();
@@ -229,6 +241,7 @@ static Bom addMetadataProperty(final String fileName, final String name, final S
bom.setMetadata(meta);
return bom;
}
+
static Bom addMetadataTools(final String fileName, final String toolName, final String version) {
Bom bom = readJSONfile(fileName);
Metadata meta = new Metadata();
@@ -240,7 +253,9 @@ static Bom addMetadataTools(final String fileName, final String toolName, final
bom.setMetadata(meta);
return bom;
}
- static Bom addComponent(final String fileName, final String compName, final String version, final String description) { // Method to store Component --> name & single name-value pair
+
+ // Method to store Component --> name & single name-value pair.
+ static Bom addComponent(final String fileName, final String compName, final String version, final String description) {
Bom bom = readJSONfile(fileName);
Component comp = new Component();
comp.setName(compName);
@@ -248,12 +263,26 @@ static Bom addComponent(final String fileName, final String compName, final Stri
comp.setType(Component.Type.FRAMEWORK);
comp.setDescription(description);
comp.setGroup("adoptium.net");
- comp.setAuthor("Adoptium Temurin");
+ comp.setAuthor("Eclipse Temurin");
comp.setPublisher("Eclipse Temurin");
bom.addComponent(comp);
return bom;
}
- static Bom addComponentProperty(final String fileName, final String compName, final String name, final String value) { // Method to add Component --> Property --> name-value pairs
+
+ static Bom addComponentHash(final String fileName, final String compName, final String hash) {
+ Bom bom = readJSONfile(fileName);
+ List componentArrayList = bom.getComponents();
+ for (Component item : componentArrayList) {
+ if (item.getName().equals(compName)) {
+ Hash hash1 = new Hash(Hash.Algorithm.SHA_256, hash);
+ item.addHash(hash1);
+ }
+ }
+ return bom;
+ }
+
+ // Method to add Component --> Property --> name-value pairs.
+ static Bom addComponentProperty(final String fileName, final String compName, final String name, final String value) {
Bom bom = readJSONfile(fileName);
List componentArrayList = bom.getComponents();
for (Component item : componentArrayList) {
@@ -266,21 +295,25 @@ static Bom addComponentProperty(final String fileName, final String compName, fi
}
return bom;
}
- static Bom addExternalReference(final String fileName, final String hashes, final String url, final String comment) { // Method to store externalReferences: dependency_version_alsa
+
+ // Method to store externalReferences: dependency_version_alsa.
+ static Bom addExternalReference(final String fileName, final String hash, final String url, final String comment) {
Bom bom = readJSONfile(fileName);
ExternalReference extRef = new ExternalReference();
- Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hashes);
+ Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hash);
extRef.setType(ExternalReference.Type.BUILD_SYSTEM); //required
- extRef.setUrl(url); // required must be a valid URL with protocal
+ extRef.setUrl(url); // required must be a valid URL with protocol
extRef.setComment(comment);
extRef.addHash(hash1);
bom.addExternalReference(extRef);
return bom;
}
- static Bom addComponentExternalReference(final String fileName, final String hashes, final String url, final String comment) { // Method to store externalReferences to store: openjdk_source
+
+ // Method to store externalReferences to store: openjdk_source.
+ static Bom addComponentExternalReference(final String fileName, final String hash, final String url, final String comment) {
Bom bom = readJSONfile(fileName);
ExternalReference extRef = new ExternalReference();
- Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hashes);
+ Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hash);
Component comp = new Component();
extRef.addHash(hash1);
extRef.setUrl(url);
@@ -366,7 +399,8 @@ static String generateBomJson(final Bom bom) {
return json;
}
- static void writeJSONfile(final Bom bom, final String fileName) { // Creates testJson.json file
+ // Writes the BOM object to the specified file.
+ static void writeJSONfile(final Bom bom, final String fileName) {
FileWriter file;
String json = generateBomJson(bom);
try {
@@ -378,7 +412,8 @@ static void writeJSONfile(final Bom bom, final String fileName) { // Cr
}
}
- static Bom readJSONfile(final String fileName) { // Returns parse bom
+ // Returns a parsed BOM object from the specified file.
+ static Bom readJSONfile(final String fileName) {
Bom bom = null;
try {
FileReader reader = new FileReader(fileName);
diff --git a/sbin/build.sh b/sbin/build.sh
index d31e74590..146be36a2 100755
--- a/sbin/build.sh
+++ b/sbin/build.sh
@@ -669,7 +669,7 @@ createSourceArchive() {
# shellcheck disable=SC2001
srcArchiveName="$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]}" | sed 's/_x64_linux_hotspot_/-sources_/g')"
else
- srcArchiveName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-sources}")
+ srcArchiveName=$(getTargetFileNameForComponent "sources")
fi
local oldPwd="${PWD}"
@@ -791,6 +791,11 @@ setupAntEnv() {
echo "Unable to find a suitable JAVA_HOME to build the cyclonedx-lib"
exit 2
fi
+
+ if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
+ javaHome=$(cygpath -w "${javaHome}")
+ fi
+
echo "${javaHome}"
}
@@ -808,14 +813,12 @@ buildCyclonedxLib() {
JAVA_HOME=${javaHome} ant -f "${ANTBUILDFILE}" build
}
-# Generate the SBoM
-generateSBoM() {
- local javaHome="${1}"
+# get the classpath to run the CycloneDX java app TemurinGenSBOM
+getCyclonedxClasspath() {
- # classpath to run CycloneDX java app TemurinGenSBOM
- CYCLONEDB_JAR_DIR="${CYCLONEDB_DIR}/build/jar"
- classpath="${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar:${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar:${CYCLONEDB_JAR_DIR}/jackson-core.jar:${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar:${CYCLONEDB_JAR_DIR}/jackson-databind.jar:${CYCLONEDB_JAR_DIR}/jackson-annotations.jar:${CYCLONEDB_JAR_DIR}/json-schema.jar:${CYCLONEDB_JAR_DIR}/commons-codec.jar:${CYCLONEDB_JAR_DIR}/commons-io.jar:${CYCLONEDB_JAR_DIR}/github-package-url.jar"
- sbomJson="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/sbom.json"
+ local CYCLONEDB_JAR_DIR="${CYCLONEDB_DIR}/build/jar"
+
+ local classpath="${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar:${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar:${CYCLONEDB_JAR_DIR}/jackson-core.jar:${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar:${CYCLONEDB_JAR_DIR}/jackson-databind.jar:${CYCLONEDB_JAR_DIR}/jackson-annotations.jar:${CYCLONEDB_JAR_DIR}/json-schema.jar:${CYCLONEDB_JAR_DIR}/commons-codec.jar:${CYCLONEDB_JAR_DIR}/commons-io.jar:${CYCLONEDB_JAR_DIR}/github-package-url.jar"
if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
classpath=""
for jarfile in "${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar" "${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar" \
@@ -826,17 +829,40 @@ generateSBoM() {
do
classpath+=$(cygpath -w "${jarfile}")";"
done
- sbomJson=$(cygpath -w "${sbomJson}")
- javaHome=$(cygpath -w "${javaHome}")
fi
+ echo "${classpath}"
+}
+
+# Generate the SBoM
+generateSBoM() {
+ if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "false" ]] || [[ ! -d "${CYCLONEDB_DIR}" ]]; then
+ echo "Skip generating SBOM"
+ return
+ fi
+
+ local javaHome="$(setupAntEnv)"
+
+ buildCyclonedxLib "${javaHome}"
+ # classpath to run java app TemurinGenSBOM
+ local classpath="$(getCyclonedxClasspath)"
+
+ local sbomTargetName=$(getTargetFileNameForComponent "sbom")
+ # Remove the tarball / zip extension from the name to be used for the SBOM
+ if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
+ sbomTargetName=$(echo "${sbomTargetName}.json" | sed "s/\.zip//")
+ else
+ sbomTargetName=$(echo "${sbomTargetName}.json" | sed "s/\.tar\.gz//")
+ fi
+
+ local sbomJson="$(joinPathOS ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} ${sbomTargetName})"
+ echo "OpenJDK SBOM will be ${sbomJson}."
+
# Clean any old json
rm -f "${sbomJson}"
- # Run a series of SBOM API commands to generate the required SBOM
- JAVA_LOC="$PRODUCT_HOME/bin/java"
- local fullVer=$($JAVA_LOC -XshowSettings:properties -version 2>&1 | grep 'java.runtime.version' | sed 's/^.*= //' | tr -d '\r')
- local fullVerOutput=$($JAVA_LOC -version 2>&1)
+ local fullVer=$(cat "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersion.txt")
+ local fullVerOutput=$(cat "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersionOutput.txt")
# Create initial SBOM json
createSBOMFile "${javaHome}" "${classpath}" "${sbomJson}"
@@ -844,7 +870,7 @@ generateSBoM() {
addSBOMMetadata "${javaHome}" "${classpath}" "${sbomJson}"
# Create component to metadata in SBOM
- addSBOMMetadataComponent "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "framework" "${fullVer}" "Temurin JDK Component"
+ addSBOMMetadataComponent "${javaHome}" "${classpath}" "${sbomJson}" "Eclipse Temurin" "framework" "${fullVer}" "Eclipse Temurin components"
# Below add property to metadata
# Add OS full version (Kernel is covered in the first field)
@@ -897,12 +923,15 @@ generateSBoM() {
# Add FreeType 3rd party
addFreeTypeVersionInfo
# Add FreeMarker 3rd party (openj9)
- addSBOMMetadataTools "${javaHome}" "${classpath}" "${sbomJson}" "FreeMarker" "$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/dependency_version_freemarker.txt)"
+ local freemarker_version="$(joinPathOS ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} 'metadata/dependency_version_freemarker.txt')"
+ if [ -f "${freemarker_version}" ]; then
+ addSBOMMetadataTools "${javaHome}" "${classpath}" "${sbomJson}" "FreeMarker" "$(cat ${freemarker_version})"
+ fi
# Add CycloneDX versions
addCycloneDXVersions
-
+
# Add Build Docker image SHA1
- buildimagesha=$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/docker.txt)
+ local buildimagesha=$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/docker.txt)
# ${BUILD_CONFIG[USE_DOCKER]^} always set to false cannot rely on it.
if [ -n "${buildimagesha}" ] && [ "${buildimagesha}" != "N.A" ]; then
addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "Use Docker for build" "true"
@@ -911,13 +940,83 @@ generateSBoM() {
addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "Use Docker for build" "false"
fi
+ checkingToolSummary
+
+ # add individual components that have been generated in this build
+ local components=("JDK" "JRE" "SOURCES" "STATIC-LIBS" "DEBUGIMAGE" "TESTIMAGE")
+ for component in "${components[@]}"
+ do
+ local componentLowerCase=$(echo "${component}" | tr '[:upper:]' '[:lower:]')
+
+ local componentName="${component} Component"
+ # shellcheck disable=SC2001
+ local archiveName=$(getTargetFileNameForComponent "${componentLowerCase}")
+ local archiveFile="$(joinPath ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} ${archiveName})"
+
+ # special handling for static-libs, determine the glibc type that is used.
+ if [ "${component}" == "STATIC-LIBS" ]; then
+ local staticLibsVariants=("" "-glibc" "-musl")
+ for staticLibsVariant in "${staticLibsVariants[@]}"
+ do
+ # shellcheck disable=SC2001
+ archiveName=$(getTargetFileNameForComponent "static-libs${staticLibsVariant}")
+ archiveFile="$(joinPath ${BUILD_CONFIG[WORKSPACE_DIR]} ${BUILD_CONFIG[TARGET_DIR]} ${archiveName})"
+ if [ -f "${archiveFile}" ]; then
+ break
+ fi
+ done
+ fi
+
+ if [ ! -f "${archiveFile}" ]; then
+ continue
+ fi
+
+ local sha=$(sha256File "${archiveFile}")
+
+ # Create JDK Component
+ addSBOMComponent "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "${fullVer}" "${BUILD_CONFIG[BUILD_VARIANT]^} ${component} Component"
+
+ # Add SHA256 hash for the component
+ addSBOMComponentHash "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "${sha}"
+
+ # Below add different properties to JDK component
+ # Add target archive name as JDK Component Property
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Filename" "${archiveName}"
+ # Add variant as JDK Component Property
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "JDK Variant" "${BUILD_CONFIG[BUILD_VARIANT]^}"
+ # Add scmRef as JDK Component Property
+ addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "SCM Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/scmref.txt"
+ # Add OpenJDK source ref commit as JDK Component Property
+ addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "OpenJDK Source Commit" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/openjdkSource.txt"
+ # Add buildRef as JDK Component Property
+ addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Temurin Build Ref" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/buildSource.txt"
+ # Add jenkins job ID as JDK Component Property
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Builder Job Reference" "${BUILD_URL:-N.A}"
+ # Add jenkins builder (agent/machine name) as JDK Component Property
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Builder Name" "${NODE_NAME:-N.A}"
+
+ # Add build timestamp
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Build Timestamp" "${BUILD_CONFIG[BUILD_TIMESTAMP]}"
+
+ # Add Tool Summary section from configure.txt
+ addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Build Tools Summary" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/dependency_tool_sum.txt"
+ # Add builtConfig JDK Component Property, load as Json string
+ built_config=$(createConfigToJsonString)
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "Build Config" "${built_config}"
+ # Add full_version_output JDK Component Property
+ addSBOMComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "full_version_output" "${fullVerOutput}"
+ # Add makejdk_any_platform_args JDK Component Property
+ addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "makejdk_any_platform_args" "${BUILD_CONFIG[WORKSPACE_DIR]}/config/makejdk-any-platform.args"
+ # Add make_command_args JDK Component Property
+ addSBOMComponentPropertyFromFile "${javaHome}" "${classpath}" "${sbomJson}" "${componentName}" "make_command_args" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/makeCommandArg.txt"
+ done
+
# Print SBOM json
echo "CycloneDX SBOM:"
cat "${sbomJson}"
echo ""
}
-
# Generate build tools info into dependency file
checkingToolSummary() {
echo "Checking and getting Tool Summary info:"
@@ -1129,7 +1228,7 @@ printJavaVersionString() {
exit 3
elif [ "${BUILD_CONFIG[CROSSCOMPILE]}" == "true" ]; then
# job is cross compiled, so we cannot run it on the build system
- # So we leave it for now and retrive the version from a downstream job after the build
+ # So we leave it for now and retrieve the version from a downstream job after the build
echo "Warning: java version can't be run on cross compiled build system. Faking version for now..."
else
# print version string around easy to find output
@@ -1193,7 +1292,6 @@ cleanAndMoveArchiveFiles() {
local testImageTargetPath=$(getTestImageArchivePath)
local debugImageTargetPath=$(getDebugImageArchivePath)
local staticLibsImageTargetPath=$(getStaticLibsArchivePath)
- local sbomTargetPath=$(getSbomArchivePath)
echo "Moving archive content to target archive paths and cleaning unnecessary files..."
@@ -1231,15 +1329,6 @@ cleanAndMoveArchiveFiles() {
mv "${testImagePath}" "${testImageTargetPath}"
fi
- # If creating SBOM, move it to the target Sbom archive path
- if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "true" ]]; then
- local sbomJson="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/sbom.json"
- echo "moving ${sbomJson} to ${sbomTargetPath}/sbom.json"
- rm -rf "${sbomTargetPath}" || true
- mkdir "${sbomTargetPath}"
- mv "${sbomJson}" "${sbomTargetPath}"
- fi
-
# Static libs image - check if the directory exists
local staticLibsImagePath="${BUILD_CONFIG[STATIC_LIBS_IMAGE_PATH]}"
local osArch
@@ -1701,6 +1790,21 @@ createArchive() {
fi
}
+# Gets the target file name for a given component in a fail-safe way.
+getTargetFileNameForComponent() {
+ local component=$1
+ local target_file_name=${BUILD_CONFIG[TARGET_FILE_NAME]}
+
+ # check if the target file name contains a -jdk pattern to be replaced with the component.
+ if [[ "${target_file_name}" == *"-jdk"* ]]; then
+ # shellcheck disable=SC2001
+ echo "${target_file_name}" | sed "s/-jdk/-${component}/"
+ else
+ # if no pattern is found, append the component name right before the extension.
+ echo "${target_file_name}" | sed -r "s/(.+)(\.tar\.gz|\.zip)/\1-${component}\2/"
+ fi
+}
+
# Create a Tar ball
createOpenJDKTarArchive() {
local jdkTargetPath=$(getJdkArchivePath)
@@ -1708,13 +1812,12 @@ createOpenJDKTarArchive() {
local testImageTargetPath=$(getTestImageArchivePath)
local debugImageTargetPath=$(getDebugImageArchivePath)
local staticLibsImageTargetPath=$(getStaticLibsArchivePath)
- local sbomTargetPath=$(getSbomArchivePath)
echo "OpenJDK JDK path will be ${jdkTargetPath}. JRE path will be ${jreTargetPath}"
if [ -d "${jreTargetPath}" ]; then
# shellcheck disable=SC2001
- local jreName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]}" | sed 's/-jdk/-jre/')
+ local jreName=$(getTargetFileNameForComponent "jre")
# for macOS system, code sign directory before creating tar.gz file
if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "darwin" ] && [ -n "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" ]; then
codesign --options runtime --timestamp --sign "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" "${jreTargetPath}"
@@ -1722,18 +1825,18 @@ createOpenJDKTarArchive() {
createArchive "${jreTargetPath}" "${jreName}"
fi
if [ -d "${testImageTargetPath}" ]; then
- echo "OpenJDK test image path will be ${testImageTargetPath}."
- local testImageName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-testimage}")
+ local testImageName=$(getTargetFileNameForComponent "testimage")
+ echo "OpenJDK test image archive file name will be ${testImageName}."
createArchive "${testImageTargetPath}" "${testImageName}"
fi
if [ -d "${debugImageTargetPath}" ]; then
- echo "OpenJDK debug image path will be ${debugImageTargetPath}."
- local debugImageName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-debugimage}")
+ local debugImageName=$(getTargetFileNameForComponent "debugimage")
+ echo "OpenJDK debug image archive file name will be ${debugImageName}."
createArchive "${debugImageTargetPath}" "${debugImageName}"
fi
if [ -d "${staticLibsImageTargetPath}" ]; then
echo "OpenJDK static libs path will be ${staticLibsImageTargetPath}."
- local staticLibsTag="-static-libs"
+ local staticLibsTag="static-libs"
if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" = "linux" ]; then
# on Linux there might be glibc and musl variants of this
local cLib="glibc"
@@ -1745,25 +1848,10 @@ createOpenJDKTarArchive() {
staticLibsTag="${staticLibsTag}-${cLib}"
fi
# shellcheck disable=SC2001
- local staticLibsImageName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]}" | sed "s/-jdk/${staticLibsTag}/g")
+ local staticLibsImageName=$(getTargetFileNameForComponent "${staticLibsTag}")
echo "OpenJDK static libs archive file name will be ${staticLibsImageName}."
createArchive "${staticLibsImageTargetPath}" "${staticLibsImageName}"
fi
- if [ -d "${sbomTargetPath}" ]; then
- # SBOM archive artifact as a .json file
- local sbomTargetName=$(echo "${BUILD_CONFIG[TARGET_FILE_NAME]//-jdk/-sbom}.json")
-
- # Remove the tarball extension from the name to be used for the SBOM
- if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
- sbomTargetName="${sbomTargetName//\.zip/}"
- else
- sbomTargetName="${sbomTargetName//\.tar\.gz/}"
- fi
-
- local sbomArchiveTarget=${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/${sbomTargetName}
- echo "OpenJDK SBOM will be ${sbomTargetName}."
- cp "${sbomTargetPath}/sbom.json" "${sbomArchiveTarget}"
- fi
# for macOS system, code sign directory before creating tar.gz file
if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "darwin" ] && [ -n "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" ]; then
codesign --options runtime --timestamp --sign "${BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]}" "${jdkTargetPath}"
@@ -1995,6 +2083,7 @@ addInfoToJson(){
addVendorToJson
addSourceToJson # Build repository and commit SHA
addOpenJDKSourceToJson # OpenJDK repository and commit SHA
+ addProductVersionToJson
}
addVariantVersionToJson(){
@@ -2037,6 +2126,15 @@ addOpenJDKSourceToJson() { # name of git repo for which SOURCE sha is from
fi
}
+addProductVersionToJson() {
+ local JAVA_LOC="$PRODUCT_HOME/bin/java"
+ local fullVer=$(${JAVA_LOC} -XshowSettings:properties -version 2>&1 | grep 'java.runtime.version' | sed 's/^.*= //' | tr -d '\r')
+ local fullVerOutput=$(${JAVA_LOC} -version 2>&1)
+
+ echo "${fullVer}" > "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersion.txt" 2>&1
+ echo "${fullVerOutput}" > "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/productVersionOutput.txt" 2>&1
+}
+
################################################################################
loadConfigFromFile
@@ -2052,17 +2150,12 @@ if [[ "${BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]}" == "true" ]]; then
printJavaVersionString
addInfoToReleaseFile
addInfoToJson
- if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "true" ]] && [[ -d "${CYCLONEDB_DIR}" ]]; then
- javaHome="$(setupAntEnv)"
- buildCyclonedxLib "${javaHome}"
- generateSBoM "${javaHome}"
- unset javaHome
- fi
cleanAndMoveArchiveFiles
copyFreeFontForMacOS
setPlistForMacOS
addNoticeFile
createOpenJDKTarArchive
+ generateSBoM
exit 0
fi
@@ -2088,17 +2181,12 @@ if [[ "${BUILD_CONFIG[MAKE_EXPLODED]}" != "true" ]]; then
printJavaVersionString
addInfoToReleaseFile
addInfoToJson
- if [[ "${BUILD_CONFIG[CREATE_SBOM]}" == "true" ]] && [[ -d "${CYCLONEDB_DIR}" ]]; then
- javaHome="$(setupAntEnv)"
- buildCyclonedxLib "${javaHome}"
- generateSBoM "${javaHome}"
- unset javaHome
- fi
cleanAndMoveArchiveFiles
copyFreeFontForMacOS
setPlistForMacOS
addNoticeFile
createOpenJDKTarArchive
+ generateSBoM
fi
echo "build.sh : $(date +%T) : All done!"
diff --git a/sbin/common/common.sh b/sbin/common/common.sh
index 26b35418c..638da169e 100755
--- a/sbin/common/common.sh
+++ b/sbin/common/common.sh
@@ -96,6 +96,21 @@ function setDockerVolumeSuffix() {
fi
}
+# Joins multiple parts to a valid file path for the current OS
+function joinPathOS() {
+ local path=$(printf '/%s' "${@}" | sed 's|/\+|/|g')
+ if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
+ path=$(cygpath -w "${path}")
+ fi
+ echo "${path}"
+}
+
+# Joins multiple parts to a valid file path using slashes
+function joinPath() {
+ local path=$(printf '/%s' "${@}" | sed 's|/\+|/|g')
+ echo "${path}"
+}
+
# Create a Tar ball
getArchiveExtension()
{
diff --git a/sbin/common/sbom.sh b/sbin/common/sbom.sh
index 0cb426a04..4ceaf4e6e 100755
--- a/sbin/common/sbom.sh
+++ b/sbin/common/sbom.sh
@@ -23,7 +23,7 @@ verifySBOMSignature() {
"${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinSignSBOM --verifySBOMSignature --jsonFile "${jsonFile}" --publicKeyFile "${publicKeyFile}"
}
-# Set basic SBMO metadata with timestamp, authors, manufacture to ${sbomJson}
+# Set basic SBOM metadata with timestamp, authors, manufacture to ${sbomJson}
addSBOMMetadata() {
local javaHome="${1}"
local classpath="${2}"
@@ -150,6 +150,17 @@ addSBOMComponentFromFile() {
"${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinGenSBOM --addComponentProp --jsonFile "${jsonFile}" --compName "${compName}" --name "${name}" --value "${value}"
}
+# Ref: https://cyclonedx.org/docs/1.4/json/#components_items_hashes
+# Add the given sha256 hash to the given SBOM Component
+addSBOMComponentHash() {
+ local javaHome="${1}"
+ local classpath="${2}"
+ local jsonFile="${3}"
+ local compName="${4}"
+ local hash="${5}"
+ "${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinGenSBOM --addComponentHash --jsonFile "${jsonFile}" --compName "${compName}" --hash "${hash}"
+}
+
# Ref: https://cyclonedx.org/docs/1.4/json/#components_items_properties
# Add the given Property name & value to the given SBOM Component
addSBOMComponentProperty() {
diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh
index 5f44dee93..caddef32d 100644
--- a/sbin/prepareWorkspace.sh
+++ b/sbin/prepareWorkspace.sh
@@ -336,7 +336,18 @@ checkingAndDownloadingAlsa() {
# Note: the uptime command below is to aid diagnostics for this issue:
# https://github.com/adoptium/temurin-build/issues/3518#issuecomment-1792606345
uptime
- gpg --keyserver keyserver.ubuntu.com --keyserver-options timeout=300 --recv-keys "${ALSA_LIB_GPGKEYID}"
+ # Will retry command below until it passes or we've failed 10 times.
+ for i in {1..10}; do
+ if gpg --keyserver keyserver.ubuntu.com --keyserver-options timeout=300 --recv-keys "${ALSA_LIB_GPGKEYID}"; then
+ echo "gpg command has passed."
+ break
+ elif [[ ${i} -lt 10 ]]; then
+ echo "gpg recv-keys attempt has failed. Retrying after 10 second pause..."
+ sleep 10s
+ else
+ echo "ERROR: gpg recv-keys final attempt has failed. Will not try again."
+ fi
+ done
echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key "${ALSA_LIB_GPGKEYID}" trust;
gpg --verify alsa-lib.tar.bz2.sig alsa-lib.tar.bz2 || exit 1
diff --git a/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java b/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java
index baf413120..a6e103201 100644
--- a/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java
+++ b/test/functional/buildAndPackage/src/net/adoptium/test/BundledFreetypeTest.java
@@ -74,7 +74,11 @@ public void freetypeOnlyBundledOnCertainPlatforms() throws IOException {
.filter(name -> freetypePattern.matcher(name).matches())
.collect(Collectors.toSet());
- if (jdkPlatform.runsOn(OperatingSystem.MACOS)) {
+ if (jdkVersion.isNewerOrEqual(21)) {
+ // jdk-21+ uses "bundled" FreeType
+ assertTrue(freetypeFiles.size() > 0,
+ "Expected libfreetype.dylib to be bundled but it is not.");
+ } else if (jdkPlatform.runsOn(OperatingSystem.MACOS)) {
assertTrue(freetypeFiles.size() > 0,
"Expected libfreetype.dylib to be bundled but it is not.");
} else if (jdkPlatform.runsOn(OperatingSystem.WINDOWS)) {
diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD
index 6f565da6a..bf930e86d 100644
--- a/tooling/ReproducibleBuilds.MD
+++ b/tooling/ReproducibleBuilds.MD
@@ -39,7 +39,7 @@ remove any Signatures from the executeables.
## Comparable Build Tools
-1. comparable_patch.sh : Patches a JDK folder to enable "Comparable" comparison of two different Vendor JDK builds.
+comparable_patch.sh : Patches a JDK folder to enable "Comparable" comparison of two different Vendor JDK builds.
The patching process involves:
- Expanding all zips and jmods, so executeables can be processed to remove signatures prior to comaprison.
@@ -47,4 +47,71 @@ The patching process involves:
- Neutralise VS_VERSION_INFO Vendor strings (Windows).
- Remove non-comparable CRC generated uuid values, which are binary values based on the hash of the content (Windows & MacOS).
- Remove Vendor strings embedded in executables, classes and text files.
+- Remove module-info differences due to "hash" of Signed module executables
- Remove any non-deterministic build process artifact strings, like Manifest Created-By stamps.
+
+### How to setup and run comparable_patch.sh on Windows
+
+#### Tooling setup:
+
+1. The comparable patch tools, tooling/src/c/WindowsUpdateVsVersionInfo.c and src/java/temurin/tools/BinRepl.java need compiling
+before the comparable_patch.sh can be run
+
+2. Compile tooling/src/c/WindowsUpdateVsVersionInfo.c :
+
+- Ensure VS2022 SDK is installed and on PATH
+- Compile:
+ - cd tooling/src/c
+ - cl WindowsUpdateVsVersionInfo.c version.lib
+
+3. Compile src/java/temurin/tools/BinRepl.java :
+
+- Ensure suitable JDK on PATH
+- cd tooling/src/java
+- javac temurin/tools/BinRepl.java
+
+4. Setting environment within a CYGWIN shell :
+
+- For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH
+- For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH
+- For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra\~1/micros\~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH
+- For running BinRepl java : export PATH=/bin:$PATH
+
+#### Running comparable_patch.sh:
+
+1. Unzip your JDK archive into a directory (eg.jdk1)
+
+2. Run comparable_patch.sh
+
+```bash
+bash comparable_patch.sh
+```
+
+The Vendor strings and urls can be found by running your jdk's "java -XshowSettings":
+
+```java
+java -XshowSettings:
+...
+ java.vendor = Eclipse Adoptium
+ java.vendor.url = https://adoptium.net/
+ java.vendor.url.bug = https://github.com/adoptium/adoptium-support/issues
+ java.vendor.version = Temurin-21.0.1+12
+...
+```
+
+eg.
+
+```bash
+bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues"
+```
+
+3. Unzip the other Vendor JDK to compare with, say into "jdk2", and run a similar comparable_patch.sh
+for that Vendor branding
+
+4. Diff recursively the now Vendor neutralized jdk directories
+
+```bash
+diff -r jdk1 jdk2
+```
+
+The diff should be "identical" if the two Vendor JDK's are "Comparable", ie."Identical except for the Vendor Branding"
diff --git a/tooling/build_autotriage/autotriage_regexes.sh b/tooling/build_autotriage/autotriage_regexes.sh
new file mode 100644
index 000000000..9a2a5999c
--- /dev/null
+++ b/tooling/build_autotriage/autotriage_regexes.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+################################################################################
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+################################################################################
+#
+# This script produces arrays of regular expressions that match a type of failure
+# that can be searched for in the output of an Eclipse Temurin build of OpenJDK.
+#
+# Each regular expression comes paired with metadata (providing useful information)
+#
+################################################################################
+
+# declare -a arrayOfRegexes
+# declare -a arrayOfRegexMetadata
+# declare -a arrayOfRegexPreventability
+
+# TODO.
\ No newline at end of file
diff --git a/tooling/build_autotriage/build_autotriage.sh b/tooling/build_autotriage/build_autotriage.sh
new file mode 100644
index 000000000..8ab8d2e74
--- /dev/null
+++ b/tooling/build_autotriage/build_autotriage.sh
@@ -0,0 +1,279 @@
+#!/bin/bash
+# shellcheck disable=SC1091
+################################################################################
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+################################################################################
+#
+# This script turns a list of Temurin build jobs into two things:
+# 1. A markdown summary table that gives pass and fail numbers.
+# 2. A list of each failing job/subjob link, plus information that can
+# help identify the specific issue causing the failure.
+#
+################################################################################
+
+declare -a arrayOfFailedJobs
+# declare -a arrayOfFailedJobRegexs
+declare -a arrayOfAllJDKVersions
+declare -a arrayOfUs
+declare -a buildIssues
+
+headJDKVersion=9999
+
+# Imports arrayOfRegexes, arrayOfRegexMetadata, and arrayOfRegexPreventability
+. ./tooling/build_autotriage/autotriage_regexes.sh
+
+# All temurin-available platforms.
+declare -a temurinPlatforms
+# The first jdk major version on that platform.
+declare -a platformStart
+# The last jdk major version on that platform ("99" for ongoing).
+declare -a platformEnd
+
+temurinPlatforms+=("aix-ppc64"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("alpine-linux-aarch64"); platformStart+=(21); platformEnd+=(99)
+temurinPlatforms+=("alpine-linux-x64"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("linux-aarch64"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("linux-arm"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("linux-ppc64le"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("linux-s390x"); platformStart+=(11); platformEnd+=(99)
+temurinPlatforms+=("linux-x64"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("mac-aarch64"); platformStart+=(11); platformEnd+=(99)
+temurinPlatforms+=("mac-x64"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("solaris-sparcv9"); platformStart+=(8); platformEnd+=(8)
+temurinPlatforms+=("solaris-x64"); platformStart+=(8); platformEnd+=(8)
+temurinPlatforms+=("windows-x64"); platformStart+=(8); platformEnd+=(99)
+temurinPlatforms+=("windows-x86-32"); platformStart+=(8); platformEnd+=(17)
+
+errorLog() {
+ buildIssues+=("$1")
+ echo "ERROR FOUND: Issue ${#buildIssues[@]}: $1"
+}
+
+# Parses the arguments to this script.
+argumentParser() {
+ if [ "$#" -lt 1 ]; then
+ echo "Illegal argument/s. You must specify at least one jdk version."
+ exit 1
+ fi
+
+ while [[ $# -gt 0 ]]
+ do
+ if [[ ! $1 =~ ^jdk[0-9]+[u]?$ ]]; then
+ if [[ ! $1 =~ ^jdk[0-9]+head?$ ]]; then
+ echo "Script has failed. Reason: Illegal argument/s (name)."
+ echo "Correct argument format is: jdk#[u|head]"
+ exit 1
+ fi
+ fi
+
+ if [[ $1 =~ ^jdk[0-9]+$ ]]; then
+ arrayOfAllJDKVersions+=("${1:3}")
+ arrayOfUs+=("")
+ elif [[ $1 =~ ^jdk[0-9]+head$ ]]; then
+ arrayOfAllJDKVersions+=("${1:3:-4}")
+ arrayOfUs+=("")
+ headJDKVersion="${1:3:-4}"
+ else
+ arrayOfAllJDKVersions+=("${1:3:-1}")
+ arrayOfUs+=("u")
+ fi
+
+ echo "JDK version identified: ${1:3}"
+ shift
+ done
+}
+
+# Iterates over the supplied JDK versions and identifies the latest timer-triggered build URLs for each version.
+# This function then checks that we're building Eclipse Temurin on every platform we should be, and makes a list
+# of all the failing builds.
+identifyFailedBuildsInTimerPipelines() {
+ # Iterate over jdk versions.
+ echo "Iterating over jdk versions."
+ for v in "${!arrayOfAllJDKVersions[@]}"
+ do
+ # First we find the latest timer-initiated pipeline for this JDK version.
+ echo "wgetting https://trss.adoptium.net/api/getBuildHistory?buildName=openjdk${arrayOfAllJDKVersions[v]}-pipeline"
+ latestTimerPipelineRaw=$(wget -q -O - "https://trss.adoptium.net/api/getBuildHistory?buildName=openjdk${arrayOfAllJDKVersions[v]}-pipeline")
+ latestTimerPipelineRaw="${latestTimerPipelineRaw},HereIsTheEndOfAVeryLongFile"
+ latestTimerPipeline=""
+ latestTimerJenkinsJobID=""
+ oldIFS=$IFS
+ IFS=","
+ for jsonEntry in $latestTimerPipelineRaw
+ do
+ if [[ $jsonEntry =~ ^\[\{\"_id\".* ]]; then
+ latestTimerPipeline=${jsonEntry:9:-1}
+ elif [[ $jsonEntry =~ ^\{\"_id\".* ]]; then
+ latestTimerPipeline=${jsonEntry:8:-1}
+ fi
+
+ if [[ $jsonEntry =~ ^\"buildNum\"\:[0-9]+$ ]]; then
+ latestTimerJenkinsJobID=${jsonEntry:11}
+ fi
+
+ if [[ ! $jsonEntry =~ .*user.* ]]; then
+ if [[ $jsonEntry =~ ^\"startBy\"\:\"timer\"[\}]?$ ]]; then
+ break
+ elif [[ $jsonEntry =~ ^\"startBy\"\:\".*build-scripts/weekly-openjdk.* ]]; then
+ break
+ elif [[ $jsonEntry =~ ^\"startBy\"\:\".*releaseTrigger_[0-9]+ea.* ]]; then
+ break
+ fi
+ fi
+
+ if [[ $jsonEntry =~ ^HereIsTheEndOfAVeryLongFile$ ]]; then
+ errorLog "Could not find any timer/ea-tag triggered pipeline jobs for jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}. Skipping to the next jdk version."
+ continue 2
+ fi
+ done
+
+ echo "Found TRSS pipeline id for jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]} - ${latestTimerPipeline}"
+ echo "Whose URL is: https://ci.adoptium.net/job/build-scripts/job/openjdk${arrayOfAllJDKVersions[v]}-pipeline/${latestTimerJenkinsJobID}/"
+
+ # Now grab a full list of builds launched by this pipeline.
+ jdkJenkinsJobVersion="jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}"
+ if [[ ${arrayOfAllJDKVersions[v]} -eq headJDKVersion ]]; then
+ jdkJenkinsJobVersion="jdk"
+ fi
+ echo "wgetting https://trss.adoptium.net/api/getAllChildBuilds?parentId=${latestTimerPipeline}&buildNameRegex=^jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}.*temurin$"
+ listOfPipelineBuilds=$(wget -q -O - "https://trss.adoptium.net/api/getAllChildBuilds?parentId=${latestTimerPipeline}&buildNameRegex=^${jdkJenkinsJobVersion}\-.*temurin$")
+ declare -a listOfBuildNames
+ declare -a listOfBuildNums
+ declare -a listOfBuildResults
+
+ shorterListOfBuilds=""
+ for jsonEntry in $listOfPipelineBuilds
+ do
+ if [[ $jsonEntry =~ ^\"buildName\"\:.* ]]; then
+ listOfBuildNames+=("${jsonEntry:13:-1}")
+ shorterListOfBuilds+="${jsonEntry},"
+ elif [[ $jsonEntry =~ .*\"buildNum\"\.* ]]; then
+ listOfBuildNums+=("${jsonEntry:11}")
+ elif [[ $jsonEntry =~ .*\"buildResult\".* ]]; then
+ listOfBuildResults+=("${jsonEntry:15:-1}")
+ continue
+ fi
+ done
+
+ echo "That pipeline's builds have been identified. Now validating them."
+
+ IFS=$oldIFS
+
+ # Now iterate over platforms to make sure we're launching every platform we should.
+ triageThesePlatforms=","
+ for p in "${!temurinPlatforms[@]}"
+ do
+ if [[ $shorterListOfBuilds =~ .*\"buildName\"\:\"${jdkJenkinsJobVersion}\-${temurinPlatforms[p]}\-temurin\".* ]]; then
+ if [[ ${arrayOfAllJDKVersions[v]} -lt ${platformStart[p]} ]]; then
+ errorLog "Error: Platform ${temurinPlatforms[p]} should not be built for ${jdkJenkinsJobVersion}. Will not triage."
+ continue
+ fi
+ if [[ ${arrayOfAllJDKVersions[v]} -gt ${platformEnd[p]} ]]; then
+ errorLog "Error: Platform ${temurinPlatforms[p]} should not be built for ${jdkJenkinsJobVersion}. Will not triage."
+ continue
+ fi
+ else
+ if [[ ${arrayOfAllJDKVersions[v]} -ge ${platformStart[p]} ]]; then
+ if [[ ${arrayOfAllJDKVersions[v]} -le ${platformEnd[p]} ]]; then
+ errorLog "Error: Platform ${temurinPlatforms[p]} should be built for ${jdkJenkinsJobVersion}, but was not launched."
+ echo "DEBUG: Looked for this: \"buildName\":\"${jdkJenkinsJobVersion}-${temurinPlatforms[p]}-temurin\""
+ echo "DEBUG: In this: $shorterListOfBuilds"
+ echo "------"
+ continue
+ fi
+ fi
+ fi
+ # If we get to this stage of the loop, then this is a platform that was both *meant* to be built, and *was* built (or attempted).
+ triageThesePlatforms+="${jdkJenkinsJobVersion}-${temurinPlatforms[p]}-temurin,"
+ done
+
+ if [[ ${#triageThesePlatforms[@]} -gt 1 ]]; then
+ errorLog "Cannot find any valid build platforms launched by jdk ${arrayOfAllJDKVersions[v]}${arrayOfUs[v]} pipeline ${latestTimerJenkinsJobID}. Skipping to the next jdk version."
+ continue
+ fi
+ echo "Platforms validated. Identifying build numbers for these platforms: ${triageThesePlatforms:1:-1}"
+
+ for b in "${!listOfBuildNames[@]}"
+ do
+ if [[ $triageThesePlatforms =~ .*,${listOfBuildNames[$b]},.* ]]; then
+ if [[ ! ${listOfBuildResults[$b]} =~ ^SUCCESS$ ]]; then
+ if [[ ! ${listOfBuildResults[$b]} =~ ^UNSTABLE$ ]]; then
+ jdkJenkinsJobVersion="jdk${arrayOfAllJDKVersions[v]}${arrayOfUs[v]}"
+ if [[ ${arrayOfAllJDKVersions[v]} -eq headJDKVersion ]]; then
+ jdkJenkinsJobVersion="jdk"
+ fi
+ failedJobLink="https://ci.adoptium.net/job/build-scripts/job/jobs/job/${jdkJenkinsJobVersion}/job/${listOfBuildNames[$b]}/${listOfBuildNums[$b]}/"
+ echo "Identified a failed build for triage: ${failedJobLink}"
+ arrayOfFailedJobs+=("${failedJobLink}")
+ fi
+ fi
+ fi
+ done
+ echo "Build numbers found, and failures will be added to the array of builds to be triaged."
+ done
+}
+
+# Takes a single failed jenkins build job URL as a string, and identifies the source of
+# the failure if possible.
+buildFailureTriager() {
+ echo "Attempting to triage a job: ${1}"
+ echo "- Failed job: ${1}" >> build_triage_output.md
+ # Todo: Iterate over the failures found and triage them against the pending array of regexes.
+ # For now we'll put them in a tidy md-style file for issue inclusion.
+
+}
+
+startOutputFile() {
+ { echo "---";
+ echo "name: Build Issue Summary";
+ echo "about: For triaging the nightly and weekend build failures";
+ echo "title: Build Issue Summary for {{ date | date('YYYY-MM-DD') }}";
+ echo "labels: 'weekly-build-triage'";
+ echo "---";
+ echo "";
+ } >> build_triage_output.md
+}
+
+# @@@@@@@@@@@@ Script execution starts here @@@@@@@@@@@@
+
+echo "Build AutoTriage is starting."
+
+argumentParser "$@"
+
+identifyFailedBuildsInTimerPipelines
+
+startOutputFile
+
+if [[ ${#arrayOfFailedJobs[@]} -gt 0 ]]; then
+ echo "# Failed Builds" >> build_triage_output.md
+ for failedJob in "${arrayOfFailedJobs[@]}"
+ do
+ buildFailureTriager "$failedJob"
+ done
+ echo "# End of list" >> build_triage_output.md
+else
+ echo "All build jobs passed. Huzzah!"
+fi
+
+if [[ ${#buildIssues[@]} -gt 0 ]]; then
+ echo "# Script Issues" >> build_triage_output.md
+ for issueID in "${!buildIssues[@]}"
+ do
+ echo "- Issue ${issueID}: ${buildIssues[issueID]}\n" >> build_triage_output.md
+ done
+ echo "# End of Issues" >> build_triage_output.md
+fi
+
+echo "Build AutoTriage is complete."
diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh
index 88cef75ba..52003b1ba 100755
--- a/tooling/comparable_patch.sh
+++ b/tooling/comparable_patch.sh
@@ -28,6 +28,11 @@ set -eu
TEMURIN_TOOLS_BINREPL="temurin.tools.BinRepl"
+if [ "$#" -ne 6 ]; then
+ echo "Syntax: comparable_patch.sh "
+ exit 1
+fi
+
JDK_DIR="$1"
VERSION_REPL="$2"
VENDOR_NAME="$3"
@@ -36,12 +41,12 @@ VENDOR_BUG_URL="$5"
VENDOR_VM_BUG_URL="$6"
# Remove excluded files known to differ
+# NOTICE - Vendor specfic notice text file
+# cacerts - Vendors use different cacerts
+# classes.jsa, classes_nocoops.jsa - CDS archive caches will differ due to Vendor string differences
function removeExcludedFiles() {
- if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then
- excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class"
- else
- excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa"
- fi
+ excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa"
+
echo "Removing excluded files known to differ: ${excluded}"
for exclude in $excluded
do
@@ -53,36 +58,183 @@ function removeExcludedFiles() {
done
done
+ echo "Successfully removed all excluded files from ${JDK_DIR}"
+}
+
+# Normalize the following ModuleAttributes that can be ordered differently
+# depending on how the vendor has signed and re-packed the JMODs
+# - ModuleResolution:
+# - ModuleTarget:
+# java.base also requires the dependent module "hash:" values to be excluded
+# as they differ due to the Signatures
+function processModuleInfo() {
if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then
- echo "Removing java.base module-info.class, known to differ by jdk.jpackage module hash"
- rm "${JDK_DIR}/jmods/expanded_java.base.jmod/classes/module-info.class"
- rm "${JDK_DIR}/lib/modules_extracted/java.base/module-info.class"
+ echo "Normalizing ModuleAttributes order in module-info.class, converting to javap"
+
+ moduleAttr="ModuleResolution ModuleTarget"
+
+ FILES=$(find "${JDK_DIR}" -type f -name "module-info.class")
+ for f in $FILES
+ do
+ echo "javap and re-order ModuleAttributes for $f"
+ javap -v -sysinfo -l -p -c -s -constants "$f" > "$f.javap.tmp"
+ rm "$f"
+
+ cc=99
+ foundAttr=false
+ attrName=""
+ # Clear any attr tmp files
+ for attr in $moduleAttr
+ do
+ rm -f "$f.javap.$attr"
+ done
+
+ while IFS= read -r line
+ do
+ cc=$((cc+1))
+
+ # Module attr have only 1 line definition
+ if [[ "$foundAttr" = true ]] && [[ "$cc" -gt 1 ]]; then
+ foundAttr=false
+ attrName=""
+ fi
+
+ # If not processing an attr then check for attr
+ if [[ "$foundAttr" = false ]]; then
+ for attr in $moduleAttr
+ do
+ if [[ "$line" =~ .*"$attr:".* ]]; then
+ cc=0
+ foundAttr=true
+ attrName="$attr"
+ fi
+ done
+ fi
+
+ # Echo attr to attr tmp file, otherwise to tmp2
+ if [[ "$foundAttr" = true ]]; then
+ echo "$line" >> "$f.javap.$attrName"
+ else
+ echo "$line" >> "$f.javap.tmp2"
+ fi
+ done < "$f.javap.tmp"
+ rm "$f.javap.tmp"
+
+ # Remove javap Classfile and timestamp and SHA-256 hash
+ if [[ "$f" =~ .*"java.base".* ]]; then
+ grep -v "Last modified\|Classfile\|SHA-256 checksum\|hash:" "$f.javap.tmp2" > "$f.javap"
+ else
+ grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap"
+ fi
+ rm "$f.javap.tmp2"
+
+ # Append any ModuleAttr tmp files
+ for attr in $moduleAttr
+ do
+ if [[ -f "$f.javap.$attr" ]]; then
+ cat "$f.javap.$attr" >> "$f.javap"
+ fi
+ rm -f "$f.javap.$attr"
+ done
+ done
fi
- echo "Successfully removed all excluded files from ${JDK_DIR}"
+}
+
+# Process SystemModules classes to remove ModuleHashes$Builder differences due to Signatures
+# 1. javap
+# 2. search for line: // Method jdk/internal/module/ModuleHashes$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes$Builder;
+# 3. followed 3 lines later by: // String
+# 4. then remove all lines until next: invokevirtual
+# 5. remove Last modified, Classfile and SHA-256 checksum javap artefact statements
+function removeSystemModulesHashBuilderParams() {
+ # Key strings
+ moduleHashesFunction="// Method jdk/internal/module/ModuleHashes\$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes\$Builder;"
+ moduleString="// String "
+ virtualFunction="invokevirtual"
+
+ systemModules="SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class"
+ echo "Removing SystemModules ModulesHashes\$Builder differences"
+ for systemModule in $systemModules
+ do
+ FILES=$(find "${JDK_DIR}" -type f -name "$systemModule")
+ for f in $FILES
+ do
+ echo "Processing $f"
+ javap -v -sysinfo -l -p -c -s -constants "$f" > "$f.javap.tmp"
+ rm "$f"
+
+ # Remove "instruction number:" prefix, so we can just match code
+ sed -i -E "s/^[[:space:]]+[0-9]+:(.*)/\1/" "$f.javap.tmp"
+
+ cc=99
+ found=false
+ while IFS= read -r line
+ do
+ cc=$((cc+1))
+ # Detect hashForModule function
+ if [[ "$line" =~ .*"$moduleHashesFunction".* ]]; then
+ cc=0
+ fi
+ # 3rd instruction line is the Module string to confirm entry
+ if [[ "$cc" -eq 3 ]] && [[ "$line" =~ .*"$moduleString"[a-z\.]+.* ]]; then
+ found=true
+ module=$(echo "$line" | tr -s ' ' | tr -d '\r' | cut -d' ' -f6)
+ echo "==> Found $module ModuleHashes\$Builder function, skipping hash parameter"
+ fi
+ # hasForModule function section finishes upon finding invokevirtual
+ if [[ "$found" = true ]] && [[ "$line" =~ .*"$virtualFunction".* ]]; then
+ found=false
+ fi
+ if [[ "$found" = false ]]; then
+ echo "$line" >> "$f.javap.tmp2"
+ fi
+ done < "$f.javap.tmp"
+ rm "$f.javap.tmp"
+ grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap"
+ rm "$f.javap.tmp2"
+ done
+ done
+
+ echo "Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}"
}
# Remove the Windows EXE/DLL timestamps and internal VS CRC and debug repro hex values
+# The Windows PE format contains various values determined from the binary content
+# which will vary due to the different Vendor branding
+# timestamp - Used to be an actual timestamp but MSFT changed this to a checksum determined from binary content
+# checksum - A checksum value of the binary
+# reprohex - A hex UUID to identify the binary version, again generated from binary content
function removeWindowsNonComparableData() {
echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}"
FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll')
for f in $FILES
do
echo "Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f"
- rm -f dumpbin.tmp
- if ! dumpbin "$f" /ALL > dumpbin.tmp; then
- echo " FAILED == > dumpbin \"$f\" /ALL > dumpbin.tmp"
+
+ # Determine non-comparable data using dumpbin
+ dmpfile="$f.dumpbin.tmp"
+ rm -f "$dmpfile"
+ if ! dumpbin "$f" /ALL > "$dmpfile"; then
+ echo " FAILED == > dumpbin \"$f\" /ALL > $dmpfile"
exit 1
fi
- timestamp=$(grep "time date stamp" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f2)
- checksum=$(grep "checksum" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f2)
- reprohex=$(grep "${timestamp} repro" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r')
- reprohexhalf=$(grep "${timestamp} repro" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r')
+
+ # Determine non-comparable stamps and hex codes from dumpbin output
+ timestamp=$(grep "time date stamp" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2)
+ checksum=$(grep "checksum" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2)
+ reprohex=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r')
+ reprohexhalf=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r')
+ rm -f "$dmpfile"
+
+ # Neutralize reprohex string
if [ -n "$reprohex" ]; then
if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA"; then
echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\""
exit 1
fi
fi
+
+ # Neutralize timestamp hex string
hexstr="00000000"
timestamphex=${hexstr:0:-${#timestamp}}$timestamp
timestamphexLE="${timestamphex:6:2}:${timestamphex:4:2}:${timestamphex:2:2}:${timestamphex:0:2}"
@@ -97,6 +249,7 @@ function removeWindowsNonComparableData() {
fi
fi
+ # Neutralize checksum string
# Prefix checksum to 8 digits
hexstr="00000000"
checksumhex=${hexstr:0:-${#checksum}}$checksum
@@ -136,7 +289,7 @@ function removeMacOSNonComparableData() {
echo "Successfully removed all MacOS dylib non-comparable UUID from ${JDK_DIR}"
}
-# Neutralize Windows VS_VERSION_INFO CompanyName
+# Neutralize Windows VS_VERSION_INFO CompanyName from the resource compiled PE section
function neutraliseVsVersionInfo() {
echo "Updating EXE/DLL VS_VERSION_INFO in ${JDK_DIR}"
FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll')
@@ -231,38 +384,53 @@ function neutraliseReleaseFile() {
echo "Removing Vendor strings from release file ${JDK_DIR}/release"
if [[ "$OS" =~ Darwin* ]]; then
+ # Remove Vendor versions
sed -i "" "s=$VERSION_REPL==g" "${JDK_DIR}/release"
sed -i "" "s=$VENDOR_NAME==g" "${JDK_DIR}/release"
- # BUILD_INFO likely different since built on different machines
- sed -i "" "s=^BUILD_INFO.*$==g" "${JDK_DIR}/release"
-
- # BUILD_SOURCE possibly built not using temurin build scripts
- sed -i "" "s=^BUILD_SOURCE.*$==g" "${JDK_DIR}/release"
- sed -i "" "s=^BUILD_SOURCE_REPO.*$==g" "${JDK_DIR}/release"
-
- # Temurin JVM_VARIANT has capital "Hotspot"
- sed -i "" "s,^JVM_VARIANT=\"Hotspot\",JVM_VARIANT=\"hotspot\",g" "${JDK_DIR}/release"
+ # Temurin BUILD_* likely different since built on different machines and bespoke to Temurin
+ sed -i "" "/^BUILD_INFO/d" "${JDK_DIR}/release"
+ sed -i "" "/^BUILD_SOURCE/d" "${JDK_DIR}/release"
+ sed -i "" "/^BUILD_SOURCE_REPO/d" "${JDK_DIR}/release"
+
+ # Remove bespoke Temurin fields
+ sed -i "" "/^SOURCE/d" "${JDK_DIR}/release"
+ sed -i "" "/^FULL_VERSION/d" "${JDK_DIR}/release"
+ sed -i "" "/^SEMANTIC_VERSION/d" "${JDK_DIR}/release"
+ sed -i "" "/^JVM_VARIANT/d" "${JDK_DIR}/release"
+ sed -i "" "/^JVM_VERSION/d" "${JDK_DIR}/release"
+ sed -i "" "/^JVM_VARIANT/d" "${JDK_DIR}/release"
+ sed -i "" "/^IMAGE_TYPE/d" "${JDK_DIR}/release"
else
+ # Remove Vendor versions
sed -i "s=$VERSION_REPL==g" "${JDK_DIR}/release"
sed -i "s=$VENDOR_NAME==g" "${JDK_DIR}/release"
- # BUILD_INFO likely different since built on different machines
- sed -i "s=^BUILD_INFO.*$==g" "${JDK_DIR}/release"
-
- # BUILD_SOURCE possibly built not using temurin build scripts
- sed -i "s=^BUILD_SOURCE.*$==g" "${JDK_DIR}/release"
- sed -i "s=^BUILD_SOURCE_REPO.*$==g" "${JDK_DIR}/release"
-
- # Temurin JVM_VARIANT has capital "Hotspot"
- sed -i "s,^JVM_VARIANT=\"Hotspot\",JVM_VARIANT=\"hotspot\",g" "${JDK_DIR}/release"
+ # Temurin BUILD_* likely different since built on different machines and bespoke to Temurin
+ sed -i "/^BUILD_INFO/d" "${JDK_DIR}/release"
+ sed -i "/^BUILD_SOURCE/d" "${JDK_DIR}/release"
+ sed -i "/^BUILD_SOURCE_REPO/d" "${JDK_DIR}/release"
+
+ # Remove bespoke Temurin fields
+ sed -i "/^SOURCE/d" "${JDK_DIR}/release"
+ sed -i "/^FULL_VERSION/d" "${JDK_DIR}/release"
+ sed -i "/^SEMANTIC_VERSION/d" "${JDK_DIR}/release"
+ sed -i "/^JVM_VARIANT/d" "${JDK_DIR}/release"
+ sed -i "/^JVM_VERSION/d" "${JDK_DIR}/release"
+ sed -i "/^JVM_VARIANT/d" "${JDK_DIR}/release"
+ sed -i "/^IMAGE_TYPE/d" "${JDK_DIR}/release"
fi
}
-if [ "$#" -ne 6 ]; then
- echo "Syntax: cmd "
- exit 1
-fi
+# Remove some non-JDK files that some Vendors distribute
+# - NEWS : Some Vendors provide a NEWS text file
+# - demo : Not all vendors distribute the demo examples
+function removeNonJdkFiles() {
+ echo "Removing non-JDK files"
+
+ rm -f "${JDK_DIR}/NEWS"
+ rm -rf "${JDK_DIR}/demo"
+}
if [ ! -d "${JDK_DIR}" ]; then
echo "$JDK_DIR does not exist"
@@ -299,6 +467,10 @@ echo "Successfully removed all Signatures from ${JDK_DIR}"
removeExcludedFiles
+processModuleInfo
+
+removeSystemModulesHashBuilderParams
+
if [[ "$OS" =~ CYGWIN* ]]; then
neutraliseVsVersionInfo
fi
@@ -319,6 +491,8 @@ neutraliseManifests
neutraliseReleaseFile
+removeNonJdkFiles
+
echo "***********"
echo "SUCCESS :-)"
echo "***********"
diff --git a/tooling/linux_repro_build_compare.sh b/tooling/linux_repro_build_compare.sh
index d6ef0db79..17c1b9bcb 100755
--- a/tooling/linux_repro_build_compare.sh
+++ b/tooling/linux_repro_build_compare.sh
@@ -27,7 +27,7 @@ installPrereqs() {
if test -r /etc/redhat-release; then
yum install -y gcc gcc-c++ make autoconf unzip zip alsa-lib-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXrandr-devel libXi-devel
yum install -y file fontconfig fontconfig-devel systemtap-sdt-devel # Not included above ...
- yum install -y git bzip2 xz openssl pigz which # pigz/which not strictly needed but help in final compression
+ yum install -y git bzip2 xz openssl pigz which jq # pigz/which not strictly needed but help in final compression
if grep -i release.6 /etc/redhat-release; then
if [ ! -r /usr/local/bin/autoconf ]; then
curl https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz | tar xpfz - || exit 1
@@ -51,20 +51,6 @@ downloadAnt() {
fi
}
-# get the TEMURIN_VERSION form the SBOM metadata
-getTemurinVersion() {
- TEMVER_MAJOR=$(grep '"major":' "$SBOM" | tr -d ' ,' | cut -d':' -f2)
- TEMVER_MINOR=$(grep '"minor":' "$SBOM" | tr -d ' ,' | cut -d':' -f2)
- TEMVER_SECURITY=$(grep '"security":' "$SBOM" | tr -d ' ,' | cut -d':' -f2)
- TEMVER_BUILD=$(grep '"build":' "$SBOM" | tr -d ' ,' | cut -d':' -f2)
-
- TEMURIN_VERSION="$TEMVER_MAJOR"
- if [ "$TEMVER_SECURITY" != "0" ]; then
- TEMURIN_VERSION="$TEMURIN_VERSION.$TEMVER_MINOR.$TEMVER_SECURITY"
- fi
- TEMURIN_VERSION="$TEMURIN_VERSION+$TEMVER_BUILD"
-}
-
setEnvironment() {
export CC="${LOCALGCCDIR}/bin/gcc-${GCCVERSION}"
export CXX="${LOCALGCCDIR}/bin/g++-${GCCVERSION}"
@@ -105,13 +91,12 @@ downloadAnt
echo "Retrieving and parsing SBOM from $SBOM_URL"
curl -LO "$SBOM_URL"
SBOM=$(basename "$SBOM_URL")
-BOOTJDK_VERSION=$(grep configure_arguments "$SBOM" | tr ' ' \\n | grep ^Temurin- | tail -1 | cut -d- -f2)
-GCCVERSION=$(tr ' ' \\n < "$SBOM" | grep CC= | cut -d- -f2 | cut -d\\ -f1)
+BOOTJDK_VERSION=$(jq -r '.metadata.tools[] | select(.name == "BOOTJDK") | .version' "$SBOM")
+GCCVERSION=$(jq -r '.metadata.tools[] | select(.name == "GCC") | .version' "$SBOM" | sed 's/.0$//')
LOCALGCCDIR=/usr/local/gcc$(echo "$GCCVERSION" | cut -d. -f1)
-TEMURIN_BUILD_SHA=$(awk -F'"' '/buildRef/{print$4}' "$SBOM" | cut -d/ -f7)
-TEMURIN_BUILD_ARGS=$(grep makejdk_any_platform_args "$SBOM" | cut -d\" -f4 | sed -e "s/--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt/'--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt'/" -e "s/ --disable-warnings-as-errors --enable-dtrace/ '--disable-warnings-as-errors --enable-dtrace'/" -e 's/\\n//g' -e "s,--jdk-boot-dir [^ ]*,--jdk-boot-dir /usr/lib/jvm/jdk-$BOOTJDK_VERSION,g")
-
-getTemurinVersion
+TEMURIN_BUILD_SHA=$(jq -r '.components[] | .properties[] | select (.name == "Temurin Build Ref") | .value' "$SBOM" | awk -F/ '{print $NF}')
+TEMURIN_BUILD_ARGS=$(jq -r '.components[] | .properties[] | select (.name == "makejdk_any_platform_args") | .value' "$SBOM" | cut -d\" -f4 | sed -e "s/--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt/'--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt'/" -e "s/ --disable-warnings-as-errors --enable-dtrace/ '--disable-warnings-as-errors --enable-dtrace'/" -e 's/\\n//g' -e "s,--jdk-boot-dir [^ ]*,--jdk-boot-dir /usr/lib/jvm/jdk-$BOOTJDK_VERSION,g")
+TEMURIN_VERSION=$(jq -r '.metadata.component.version' "$SBOM" | sed 's/-beta//' | cut -f1 -d"-")
NATIVE_API_ARCH=$(uname -m)
if [ "${NATIVE_API_ARCH}" = "x86_64" ]; then NATIVE_API_ARCH=x64; fi