diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72133a9a8..db5c7e130 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -133,7 +133,6 @@ jobs: matrix: os: [macOS] version: [ - { name: jdk8u, distro: macos-11 }, { name: jdk11u, distro: macos-14 }, { name: jdk17u, distro: macos-14 } ] @@ -167,21 +166,7 @@ jobs: run: | brew install automake bash binutils freetype gnu-sed nasm - - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 - id: setup-java - with: - java-version: 7 - distribution: 'zulu' - if: matrix.version.name == 'jdk8u' - - - name: Select correct Xcode (JDK8) - if: matrix.version.name == 'jdk8u' - run: | - rm -rf /Applications/Xcode.app - ln -s /Applications/Xcode_11.7.app /Applications/Xcode.app - - name: Select correct Xcode (JDK11+) - if: matrix.version.name != 'jdk8u' run: | rm -rf /Applications/Xcode.app ln -s /Applications/Xcode_15.2.app /Applications/Xcode.app @@ -190,22 +175,16 @@ jobs: run: | export JAVA_HOME=$JAVA_HOME_11_X64 # jdk11u+ uses two part exploded & assemble build - if [ ${{ matrix.version.name }} != "jdk8u" ]; then - export BUILD_ARGS="--make-exploded-image --create-sbom" - ./build-farm/make-adopt-build-farm.sh - export BUILD_ARGS="--assemble-exploded-image --create-sbom" - ./build-farm/make-adopt-build-farm.sh - else - export BUILD_ARGS="--create-sbom" - ./build-farm/make-adopt-build-farm.sh - fi + export BUILD_ARGS="--make-exploded-image --create-sbom" + ./build-farm/make-adopt-build-farm.sh + export BUILD_ARGS="--assemble-exploded-image --create-sbom" + ./build-farm/make-adopt-build-farm.sh env: JAVA_TO_BUILD: ${{ matrix.version.name }} ARCHITECTURE: x64 VARIANT: ${{ matrix.variant }} TARGET_OS: mac FILENAME: OpenJDK.tar.gz - JDK7_BOOT_DIR: ${{ steps.setup-java.outputs.path }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 name: Collect and Archive Artifacts diff --git a/build-farm/platform-specific-configurations/aix.sh b/build-farm/platform-specific-configurations/aix.sh index 615f0b7cc..78ff55677 100755 --- a/build-farm/platform-specific-configurations/aix.sh +++ b/build-farm/platform-specific-configurations/aix.sh @@ -48,7 +48,7 @@ source "$SCRIPT_DIR/../../sbin/common/constants.sh" export PATH="/opt/freeware/bin:/usr/local/bin:/opt/IBM/xlC/13.1.3/bin:/opt/IBM/xlc/13.1.3/bin:$PATH" # Without this, java adds /usr/lib to the LIBPATH and it's own library # directories of anything it forks which breaks linkage -export LIBPATH=/opt/freeware/lib/pthread/ppc64:/opt/freeware/lib:/usr/lib +export LIBPATH=/opt/freeware/lib/pthread:/opt/freeware/lib:/usr/lib export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-cups-include=/opt/freeware/include" # Any version below 11 @@ -127,13 +127,19 @@ else export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} DF=/usr/sysv/bin/df" fi -if [ "$JAVA_FEATURE_VERSION" -ge 11 ]; then +if [ "$JAVA_FEATURE_VERSION" -le 21 ] && [ "$JAVA_FEATURE_VERSION" -ge 11 ]; then export LANG=C export PATH=/opt/freeware/bin:$JAVA_HOME/bin:/usr/local/bin:/opt/IBM/xlC/16.1.0/bin:/opt/IBM/xlc/16.1.0/bin:$PATH export CC=xlclang export CXX=xlclang++ fi - +if [ "$JAVA_FEATURE_VERSION" -ge 22 ]; then + export PATH=/opt/freeware/bin:$JAVA_HOME/bin:/usr/local/bin:/opt/IBM/openxlC/17.1.1/bin:$PATH + export EXTRA_PATH=/opt/IBM/openxlC/17.1.1/tools + export TOOLCHAIN_TYPE="clang" + export CC=ibm-clang_r + export CXX=ibm-clang++_r +fi # J9 JDK14 builds seem to be chewing up more RAM than the others, so restrict it # Ref: https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1151 if [ "$JAVA_FEATURE_VERSION" -ge 14 ]; then diff --git a/configureBuild.sh b/configureBuild.sh index e3e8acf98..a16dde234 100755 --- a/configureBuild.sh +++ b/configureBuild.sh @@ -1,5 +1,5 @@ #!/bin/bash -# shellcheck disable=SC1091 +# shellcheck disable=SC1091,SC2155 # ******************************************************************************** # Copyright (c) 2018 Contributors to the Eclipse Foundation # @@ -170,6 +170,7 @@ setVariablesForConfigure() { setRepository() { local suffix + local githubRepoName=$(getOpenjdkGithubRepoName "${BUILD_CONFIG[OPENJDK_FOREST_NAME]}") # Location of Extensions for OpenJ9 project if [[ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_OPENJ9}" ]]; then @@ -200,12 +201,12 @@ setRepository() { || [ "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" == "${JDK11_CORE_VERSION}" ]; then suffix="openjdk/riscv-port-${BUILD_CONFIG[OPENJDK_FOREST_NAME]}" else - suffix="openjdk/${BUILD_CONFIG[OPENJDK_FOREST_NAME]}" + suffix="openjdk/${githubRepoName}" fi elif [[ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_TEMURIN}" ]]; then - suffix="adoptium/${BUILD_CONFIG[OPENJDK_FOREST_NAME]}" + suffix="adoptium/${githubRepoName}" else - suffix="openjdk/${BUILD_CONFIG[OPENJDK_FOREST_NAME]}" + suffix="openjdk/${githubRepoName}" fi local repository @@ -219,6 +220,30 @@ setRepository() { repository="$(echo "${repository}" | awk '{print tolower($0)}')" BUILD_CONFIG[REPOSITORY]="${BUILD_CONFIG[REPOSITORY]:-${repository}}" + + echo "Using source repository ${BUILD_CONFIG[REPOSITORY]}" +} + +# Given a forest_name (eg.jdk23), return the corresponding repository name +getOpenjdkGithubRepoName() { + local forest_name="$1" + local repoName="" + + # "Update" versions are currently in a repository with the name of the forest + if [[ ${forest_name} == *u ]]; then + repoName="${forest_name}" + else + local featureNumber=$(echo "${forest_name}" | tr -d "[:alpha:]") + + # jdk-23+ stabilisation versions are within the jdk(head) repository + if [[ "${featureNumber}" -ge 23 ]]; then + repoName="jdk" + else + repoName="${forest_name}" + fi + fi + + echo "${repoName}" } # Specific architectures need to have special build settings diff --git a/sbin/build.sh b/sbin/build.sh index 0775aac50..21c5f49dd 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -132,7 +132,7 @@ configureReproducibleBuildParameter() { # Ensure reproducible and comparable binary with a unique build user identifier addConfigureArg "--with-build-user=" "admin" - if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "aix" ]; then + if [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "aix" ] && [ "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" -lt 22 ]; then addConfigureArg "--with-extra-cflags=" "-qnotimestamps" addConfigureArg "--with-extra-cxxflags=" "-qnotimestamps" fi @@ -870,7 +870,10 @@ generateSBoM() { return fi - local javaHome="$(setupAntEnv)" + # exit from local var=$(setupAntEnv) is not propagated. We have to ensure that the exit propagates, and is fatal for the script + # So the declaration is split. In that case the bug does not occur and thus the `exit 2` from setupAntEnv is correctly propagated + local javaHome + javaHome="$(setupAntEnv)" buildCyclonedxLib "${javaHome}" # classpath to run java app TemurinGenSBOM @@ -2171,7 +2174,7 @@ addJVMVariant() { } addBuildSHA() { # git SHA of the build repository i.e. openjdk-build - local buildSHA=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}" rev-parse HEAD 2>/dev/null) + local buildSHA=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}" && git rev-parse HEAD 2>/dev/null) if [[ $buildSHA ]]; then # shellcheck disable=SC2086 echo -e BUILD_SOURCE=\"git:$buildSHA\" >>release @@ -2181,7 +2184,7 @@ addBuildSHA() { # git SHA of the build repository i.e. openjdk-build } addBuildSourceRepo() { # name of git repo for which BUILD_SOURCE sha is from - local buildSourceRepo=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}" config --get remote.origin.url 2>/dev/null) + local buildSourceRepo=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}" && git config --get remote.origin.url 2>/dev/null) if [[ $buildSourceRepo ]]; then # shellcheck disable=SC2086 echo -e BUILD_SOURCE_REPO=\"$buildSourceRepo\" >>release @@ -2191,7 +2194,7 @@ addBuildSourceRepo() { # name of git repo for which BUILD_SOURCE sha is from } addOpenJDKSourceRepo() { # name of git repo for which SOURCE sha is from - local openjdkSourceRepo=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" config --get remote.origin.url 2>/dev/null) + local openjdkSourceRepo=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" && git config --get remote.origin.url 2>/dev/null) if [[ $openjdkSourceRepo ]]; then # shellcheck disable=SC2086 echo -e SOURCE_REPO=\"$openjdkSourceRepo\" >>release @@ -2221,7 +2224,7 @@ addJ9Tag() { # This code makes sure that a version number is always present in the release file i.e. openj9-0.21.0 local j9Location="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}/openj9" # Pull the tag associated with the J9 commit being used - J9_TAG=$(git -C $j9Location describe --abbrev=0) + J9_TAG=$(cd "$j9Location" && git describe --abbrev=0) # shellcheck disable=SC2086 if [ ${BUILD_CONFIG[RELEASE]} = false ]; then echo -e OPENJ9_TAG=\"$J9_TAG\" >> release @@ -2304,9 +2307,9 @@ addVendorToJson(){ } addSourceToJson(){ # Pulls the origin repo, or uses 'openjdk-build' in rare cases of failure - local repoName=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}" config --get remote.origin.url 2>/dev/null) + local repoName=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}" && git config --get remote.origin.url 2>/dev/null) local repoNameStr="${repoName:-"ErrorUnknown"}" - local buildSHA=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}" rev-parse HEAD 2>/dev/null) + local buildSHA=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}" && git rev-parse HEAD 2>/dev/null) if [[ $buildSHA ]]; then echo -n "${repoNameStr%%.git}/commit/$buildSHA" > "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/buildSource.txt" else @@ -2315,8 +2318,8 @@ addSourceToJson(){ # Pulls the origin repo, or uses 'openjdk-build' in rare case } addOpenJDKSourceToJson() { # name of git repo for which SOURCE sha is from - local openjdkSourceRepo=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" config --get remote.origin.url 2>/dev/null) - local openjdkSHA=$(git -C "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" rev-parse HEAD 2>/dev/null) + local openjdkSourceRepo=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" && git config --get remote.origin.url 2>/dev/null) + local openjdkSHA=$(cd "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" && git rev-parse HEAD 2>/dev/null) if [[ $openjdkSHA ]]; then echo -n "${openjdkSourceRepo%%.git}/commit/${openjdkSHA}" > "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/openjdkSource.txt" else diff --git a/sbin/common/common.sh b/sbin/common/common.sh index 4b2a73f0b..8b2ea759c 100755 --- a/sbin/common/common.sh +++ b/sbin/common/common.sh @@ -15,8 +15,17 @@ # shellcheck disable=SC2155 # shellcheck disable=SC2153 function setOpenJdkVersion() { + # forest_name represents the JDK version with "u" suffix for an "update version" + # + # It no longer relates directly to the openjdk repository name, as the jdk(head) repository + # now has version branches for jdk-23+ + # + # Format: jdkNN[u] + # local forest_name=$1 + echo "Setting version based on forest_name=${forest_name}" + # The argument passed here have actually very strict format of jdk8, jdk8u..., jdk # the build may fail later if this is not honoured. # If your repository has a different name, you can use --version or build from dir/snapshot @@ -81,6 +90,45 @@ function setOpenJdkVersion() { # feature number e.g. 11 BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]=${featureNumber} + # Set default branch based on JDK forest and feature number + setBranch +} + +# Set the default BUILD_CONFIG[BRANCH] for the jdk version being built +# For "hotspot" and "Temurin" builds of non-"u" jdk-23+ the branch is dev_ +function setBranch() { + + # Which repo branch to build, e.g. dev by default for temurin, "openj9" for openj9 + local branch="master" + local adoptium_mirror_branch="dev" + + # non-u (and non-tip) jdk-23+ hotspot and adoptium version source is within a "version" branch in the "jdk" repository + if [[ ${BUILD_CONFIG[OPENJDK_FOREST_NAME]} != *u ]] && [[ ${BUILD_CONFIG[OPENJDK_FOREST_NAME]} != "jdk" ]] && [[ "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" -ge 23 ]]; then + branch="jdk${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" + adoptium_mirror_branch="dev_jdk${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" + fi + + if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_TEMURIN}" ]; then + branch="${adoptium_mirror_branch}" + elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_OPENJ9}" ]; then + branch="openj9"; + elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_DRAGONWELL}" ]; then + branch="master"; + elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_FAST_STARTUP}" ]; then + branch="master"; + elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_CORRETTO}" ]; then + branch="develop"; + elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_BISHENG}" ]; then + if [ "${BUILD_CONFIG[OS_ARCHITECTURE]}" == "riscv64" ] ; then + branch="risc-v" + else + branch="master" + fi + fi + + BUILD_CONFIG[BRANCH]=${BUILD_CONFIG[BRANCH]:-$branch} + + echo "Default branch set to BUILD_CONFIG[BRANCH]=${BUILD_CONFIG[BRANCH]}" } function crossPlatformRealPath() { diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 2f4ff4b61..0467cbd95 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -399,33 +399,6 @@ function parseConfigurationArguments() { *) echo >&2 "Invalid build.sh option: ${opt}"; exit 1;; esac done - - setBranch -} - -function setBranch() { - - # Which repo branch to build, e.g. dev by default for temurin, "openj9" for openj9 - local branch="master" - if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_TEMURIN}" ]; then - branch="dev" - elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_OPENJ9}" ]; then - branch="openj9"; - elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_DRAGONWELL}" ]; then - branch="master"; - elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_FAST_STARTUP}" ]; then - branch="master"; - elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_CORRETTO}" ]; then - branch="develop"; - elif [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_BISHENG}" ]; then - if [ "${BUILD_CONFIG[OS_ARCHITECTURE]}" == "riscv64" ] ; then - branch="risc-v" - else - branch="master" - fi - fi - - BUILD_CONFIG[BRANCH]=${BUILD_CONFIG[BRANCH]:-$branch} } # Set the local dir if used diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh index 53952502a..886f2c00e 100644 --- a/sbin/prepareWorkspace.sh +++ b/sbin/prepareWorkspace.sh @@ -51,10 +51,11 @@ copyFromDir() { # this is workarounding --strip-components 1 missing on gnu tar # it requires absolute tar-filepath as it changes dir and is hardcoded to one # similar approach can be used also for zip in future -untarGnuAbsPathWithStripComponents1() { +# warning! this method do not merge if (parts of!) destination exists. +unpackGnuAbsPathWithStrip1Component() { local tmp=$(mktemp -d) pushd "$tmp" > /dev/null - tar "$@" + "$@" popd > /dev/null mv "$tmp"/*/* . mv "$tmp"/*/.* . || echo "no hidden files in tarball" @@ -62,18 +63,30 @@ untarGnuAbsPathWithStripComponents1() { rmdir "$tmp" } +untarGnuAbsPathWithStrip1Component() { + unpackGnuAbsPathWithStrip1Component tar -xf "$@" +} + +unzipGnuAbsPathWithStrip1Component() { + unpackGnuAbsPathWithStrip1Component unzip "$@" +} + unpackFromArchive() { echo "Extracting OpenJDK source tarball ${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]} to $(pwd)/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]} to build the binary" # If the tarball contains .git files, they should be ignored later - # todo, support also zips? pushd "./${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" - local topLevelItems=$(tar --exclude='*/*' -tf "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" | grep "/$" -c) || local topLevelItems=1 - if [ "$topLevelItems" -eq "1" ] ; then - echo "Source tarball contains exactly one directory" - untarGnuAbsPathWithStripComponents1 -xf "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" + if [ "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]: -4}" == ".zip" ] ; then + echo "Source zip unpacked as if it contains exactly one directory" + unzipGnuAbsPathWithStrip1Component "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" else - echo "Source tarball does not contain a top level directory" - tar -xf "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" + local topLevelItems=$(tar --exclude='*/*' -tf "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" | grep "/$" -c) || local topLevelItems=1 + if [ "$topLevelItems" -eq "1" ] ; then + echo "Source tarball contains exactly one directory" + untarGnuAbsPathWithStrip1Component "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" + else + echo "Source tarball does not contain a top level directory" + tar -xf "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}" + fi fi rm -rf "build" popd @@ -843,13 +856,13 @@ applyPatches() { createSourceTagFile(){ if [ "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" == "${JDK8_CORE_VERSION}" ]; then local OpenJDK_TopDir="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}" - local OpenJDK_SHA=$(git -C "$OpenJDK_TopDir" rev-parse --short HEAD) + local OpenJDK_SHA=$(cd "$OpenJDK_TopDir" && git rev-parse --short HEAD) if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_OPENJ9}" ]; then # OpenJ9 list 3 SHA's in their release file: OpenJDK, OpenJ9, and OMR. local OpenJ9_TopDir="$OpenJDK_TopDir/openj9" local OMR_TopDir="$OpenJDK_TopDir/omr" - local OpenJ9_SHA=$(git -C "$OpenJ9_TopDir" rev-parse --short HEAD) - local OMR_SHA=$(git -C "$OMR_TopDir" rev-parse --short HEAD) + local OpenJ9_SHA=$(cd "$OpenJ9_TopDir" && git rev-parse --short HEAD) + local OMR_SHA=$(cd "$OMR_TopDir" && git rev-parse --short HEAD) (printf "OpenJDK: %s OpenJ9: %s OMR: %s" "$OpenJDK_SHA" "$OpenJ9_SHA" "$OMR_SHA") > "$OpenJDK_TopDir/.hgtip" else # Other variants only list the main repo SHA. (printf "OpenJDK: %s" "$OpenJDK_SHA") > "$OpenJDK_TopDir/.hgtip" diff --git a/test/system/reproducibleCompare/playlist.xml b/test/system/reproducibleCompare/playlist.xml index ddae35c60..bbeff146c 100644 --- a/test/system/reproducibleCompare/playlist.xml +++ b/test/system/reproducibleCompare/playlist.xml @@ -18,10 +18,12 @@ reproducible.mk Rebuild_Same_JDK_Reproducibility_Test - docker run -v "$(TEST_RESROOT):/home/jenkins/test" -w "/home/jenkins/" -v "$(TEST_JDK_HOME):/home/jenkins/jdkbinary/" --name reproducibleCompare centos:7 /bin/bash /home/jenkins/test/linux_repro_build_compare.sh $(SBOM_FILE) /home/jenkins/jdkbinary; \ + docker container rm reproducibleCompare; \ + docker run -v "$(TEST_RESROOT):/home/jenkins/test" -w "/home/jenkins/" -v "$(TEST_JDK_HOME):/home/jenkins/jdkbinary/" --name reproducibleCompare centos:7 /bin/bash /home/jenkins/test/linux_repro_build_compare.sh $(SBOM_FILE) /home/jenkins/jdkbinary; \ $(TEST_STATUS); \ docker cp reproducibleCompare:/home/jenkins/reprotest.diff ./; \ docker cp reproducibleCompare:/home/jenkins/reproJDK.tar.gz ./; \ + docker cp reproducibleCompare:/home/jenkins/SBOM.json ./; \ docker container rm reproducibleCompare diff --git a/tooling/reproducible/linux_repro_build_compare.sh b/tooling/reproducible/linux_repro_build_compare.sh index 7651ac753..57e6f0f58 100755 --- a/tooling/reproducible/linux_repro_build_compare.sh +++ b/tooling/reproducible/linux_repro_build_compare.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # ******************************************************************************** # Copyright (c) 2024 Contributors to the Eclipse Foundation # @@ -25,7 +25,7 @@ ANT_SHA=9028e2fc64491cca0f991acc09b06ee7fe644afe41d1d6caf72702ca25c4613c ANT_CONTRIB_VERSION=1.0b3 ANT_CONTRIB_SHA=4d93e07ae6479049bb28071b069b7107322adaee5b70016674a0bffd4aac47f9 isJdkDir=false - +USING_DEVKIT="false" installPrereqs() { if test -r /etc/redhat-release; then yum install -y gcc gcc-c++ make autoconf unzip zip alsa-lib-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXrandr-devel libXi-devel @@ -75,28 +75,50 @@ downloadAnt() { fi } -setEnvironment() { +setNonDevkitGccEnvironment() { export CC="${LOCALGCCDIR}/bin/gcc-${GCCVERSION}" export CXX="${LOCALGCCDIR}/bin/g++-${GCCVERSION}" export LD_LIBRARY_PATH="${LOCALGCCDIR}/lib64" - # /usr/local/bin required to pick up the new autoconf if required +} + +setAntEnvironment() { export PATH="${LOCALGCCDIR}/bin:/usr/local/bin:/usr/bin:$PATH:/usr/local/apache-ant-${ANT_VERSION}/bin" - ls -ld "$CC" "$CXX" "/usr/lib/jvm/jdk-${BOOTJDK_VERSION}/bin/javac" || exit 1 } cleanBuildInfo() { - # shellcheck disable=SC3043 local DIR="$1" # BUILD_INFO name of OS level build was built on will likely differ sed -i '/^BUILD_INFO=.*$/d' "${DIR}/release" } +setTemurinBuildArgs() { + local buildArgs="$1" + local bootJdk="$2" + local timeStamp="$3" + local ignoreOptions=("--enable-sbom-strace ") + for ignoreOption in "${ignoreOptions[@]}"; do + buildArgs="${buildArgs/${ignoreOption}/}" + done + # set --build-reproducible-date if not yet + if [[ "${buildArgs}" != *"--build-reproducible-date"* ]]; then + buildArgs="--build-reproducible-date \"${timeStamp}\" ${buildArgs}" + fi + #reset --jdk-boot-dir + # shellcheck disable=SC2001 + buildArgs="$(echo "$buildArgs" | sed -e "s|--jdk-boot-dir [^ ]*|--jdk-boot-dir /usr/lib/jvm/jdk-${bootJdk}|")" + echo "${buildArgs}" +} + downloadTooling() { + local using_DEVKIT=$1 + if [ ! -r "/usr/lib/jvm/jdk-${BOOTJDK_VERSION}/bin/javac" ]; then echo "Retrieving boot JDK $BOOTJDK_VERSION" && mkdir -p /usr/lib/jvm && curl -L "https://api.adoptium.net/v3/binary/version/jdk-${BOOTJDK_VERSION}/linux/${NATIVE_API_ARCH}/jdk/hotspot/normal/eclipse?project=jdk" | (cd /usr/lib/jvm && tar xpzf -) fi - if [ ! -r "${LOCALGCCDIR}/bin/g++-${GCCVERSION}" ]; then - echo "Retrieving gcc $GCCVERSION" && curl "https://ci.adoptium.net/userContent/gcc/gcc$(echo "$GCCVERSION" | tr -d .).$(uname -m).tar.xz" | (cd /usr/local && tar xJpf -) || exit 1 + if [[ "${using_DEVKIT}" == "false" ]]; then + if [ ! -r "${LOCALGCCDIR}/bin/g++-${GCCVERSION}" ]; then + echo "Retrieving gcc $GCCVERSION" && curl "https://ci.adoptium.net/userContent/gcc/gcc$(echo "$GCCVERSION" | tr -d .).$(uname -m).tar.xz" | (cd /usr/local && tar xJpf -) || exit 1 + fi fi if [ ! -r temurin-build ]; then git clone https://github.com/adoptium/temurin-build || exit 1 @@ -113,7 +135,6 @@ checkAllVariablesSet() { installPrereqs downloadAnt -# shellcheck disable=SC3010 if [[ $SBOM_PARAM =~ ^https?:// ]]; then echo "Retrieving and parsing SBOM from $SBOM_PARAM" curl -LO "$SBOM_PARAM" @@ -126,23 +147,41 @@ BOOTJDK_VERSION=$(jq -r '.metadata.tools[] | select(.name == "BOOTJDK") | .versi GCCVERSION=$(jq -r '.metadata.tools[] | select(.name == "GCC") | .version' "$SBOM" | sed 's/.0$//') LOCALGCCDIR=/usr/local/gcc$(echo "$GCCVERSION" | cut -d. -f1) TEMURIN_BUILD_SHA=$(jq -r '.components[0] | .properties[] | select (.name == "Temurin Build Ref") | .value' "$SBOM" | awk -F/ '{print $NF}') -TEMURIN_BUILD_ARGS=$(jq -r '.components[0] | .properties[] | select (.name == "makejdk_any_platform_args") | .value' "$SBOM" | cut -d\" -f4 | sed -e "s/--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt/'--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt'/" -e "s/ --disable-warnings-as-errors --enable-dtrace/ '--disable-warnings-as-errors --enable-dtrace'/" -e 's/\\n//g' -e "s,--jdk-boot-dir [^ ]*,--jdk-boot-dir /usr/lib/jvm/jdk-$BOOTJDK_VERSION,g") TEMURIN_VERSION=$(jq -r '.metadata.component.version' "$SBOM" | sed 's/-beta//' | cut -f1 -d"-") +BUILDSTAMP=$(jq -r '.components[0].properties[] | select(.name == "Build Timestamp") | .value' "$SBOM") +TEMURIN_BUILD_ARGS=$(jq -r '.components[0] | .properties[] | select (.name == "makejdk_any_platform_args") | .value' "$SBOM") + +# Using old method to escape configure-args in SBOM (build PR 3835)", can be removed later. +if echo "$TEMURIN_BUILD_ARGS" | grep -v configure-args\ \\\"; then + TEMURIN_BUILD_ARGS=$(echo "$TEMURIN_BUILD_ARGS" | \ + cut -d\" -f4 | \ + sed -e "s/--with-jobs=[0-9]*//g" \ + -e "s/--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt/'--disable-warnings-as-errors --enable-dtrace --without-version-pre --without-version-opt'/" \ + -e "s/--disable-warnings-as-errors --enable-dtrace *--with-version-opt=ea/'--disable-warnings-as-errors --enable-dtrace --with-version-opt=ea'/" \ + -e "s/ --disable-warnings-as-errors --enable-dtrace/ '--disable-warnings-as-errors --enable-dtrace'/" \ + -e 's/\\n//g') +fi NATIVE_API_ARCH=$(uname -m) if [ "${NATIVE_API_ARCH}" = "x86_64" ]; then NATIVE_API_ARCH=x64; fi if [ "${NATIVE_API_ARCH}" = "armv7l" ]; then NATIVE_API_ARCH=arm; fi +if [[ "$TEMURIN_BUILD_ARGS" =~ .*"--use-adoptium-devkit".* ]]; then + USING_DEVKIT="true" +fi checkAllVariablesSet - -downloadTooling -setEnvironment +downloadTooling "$USING_DEVKIT" +if [[ "${USING_DEVKIT}" == "false" ]]; then + setNonDevkitGccEnvironment +fi +setAntEnvironment +echo "original temurin build args is ${TEMURIN_BUILD_ARGS}" +TEMURIN_BUILD_ARGS=$(setTemurinBuildArgs "$TEMURIN_BUILD_ARGS" "$BOOTJDK_VERSION" "$BUILDSTAMP") if [ -z "$JDK_PARAM" ] && [ ! -d "jdk-${TEMURIN_VERSION}" ] ; then JDK_PARAM="https://api.adoptium.net/v3/binary/version/jdk-${TEMURIN_VERSION}/linux/${NATIVE_API_ARCH}/jdk/hotspot/normal/eclipse?project=jdk" fi -# shellcheck disable=SC3010 if [[ $JDK_PARAM =~ ^https?:// ]]; then echo Retrieving original tarball from adoptium.net && curl -L "$JDK_PARAM" | tar xpfz - && ls -lart "$PWD/jdk-${TEMURIN_VERSION}" || exit 1 elif [[ $JDK_PARAM =~ tar.gz ]]; then @@ -158,20 +197,23 @@ if [ "${isJdkDir}" = true ]; then comparedDir=$JDK_PARAM fi +echo "Rebuild args for makejdk_any_platform.sh are: $TEMURIN_BUILD_ARGS" echo " cd temurin-build && ./makejdk-any-platform.sh $TEMURIN_BUILD_ARGS 2>&1 | tee build.$$.log" | sh + echo Comparing ... mkdir compare.$$ tar xpfz temurin-build/workspace/target/OpenJDK*-jdk_*tar.gz -C compare.$$ cp temurin-build/workspace/target/OpenJDK*-jdk_*tar.gz reproJDK.tar.gz +cp "$SBOM" SBOM.json cleanBuildInfo "${comparedDir}" cleanBuildInfo "compare.$$/jdk-$TEMURIN_VERSION" - rc=0 + # shellcheck disable=SC2069 diff -r "${comparedDir}" "compare.$$/jdk-$TEMURIN_VERSION" 2>&1 > "reprotest.diff" || rc=$? -if [ $rc = 0 ]; then +if [ $rc -eq 0 ]; then echo "Compare identical !" else cat "reprotest.diff"