diff --git a/README.md b/README.md index d4df9e6b84..f9ced97534 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,11 @@ For reference, OpenJDK version numbers look like 1.8.0_162-b12 (for Java 8) or (162) or the 3rd position in the semVer version string (4). This is typically used in conjunction with -b. +--use-adoptium-devkit +Download and use the given DevKit from https://github.com/adoptium/devkit-binaries/releases. +The DevKit is downloaded and unpacked to WORKSPACE_DIR/WORKING_DIR/devkit +and will add the configure arg --with-devkit=WORKSPACE_DIR/WORKING_DIR/devkit. + --use-jep319-certs Use certs defined in JEP319 in Java 8/9. Deprecated, has no effect. diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh index 82bdbe3e41..4ef449e9ea 100755 --- a/build-farm/platform-specific-configurations/linux.sh +++ b/build-farm/platform-specific-configurations/linux.sh @@ -332,6 +332,8 @@ fi if [[ "${CONFIGURE_ARGS}" =~ .*"--with-devkit=".* ]]; then echo "Using gcc from DevKit toolchain specified in configure args" +elif [[ "${BUILD_ARGS}" =~ .*"--use-adoptium-devkit".* ]]; then + echo "Using gcc from Adoptium DevKit toolchain specified in --use-adoptium-devkit build args" else if [ "${VARIANT}" == "${BUILD_VARIANT_DRAGONWELL}" ] && [ "$JAVA_FEATURE_VERSION" -eq 11 ] && [ -r /usr/local/gcc9/ ] && [ "${ARCHITECTURE}" == "aarch64" ]; then # GCC9 rather than 10 requested by Alibaba for now diff --git a/sbin/build.sh b/sbin/build.sh index 545ad3bfc2..9b9fec12ed 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -73,6 +73,13 @@ addConfigureArgIfValueIsNotEmpty() { fi } +# Configure the DevKit if required +configureDevKitConfigureParameter() { + if [[ -n "${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" ]]; then + addConfigureArg "--with-devkit=" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit" + fi +} + # Configure the boot JDK configureBootJDKConfigureParameter() { addConfigureArgIfValueIsNotEmpty "--with-boot-jdk=" "${BUILD_CONFIG[JDK_BOOT_DIR]}" @@ -534,6 +541,7 @@ configureZlibLocation() { configureCommandParameters() { configureVersionStringParameter configureBootJDKConfigureParameter + configureDevKitConfigureParameter configureShenandoahBuildParameter configureMacOSCodesignParameter configureDebugParameters diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 88f22bcfcb..26dba69e09 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -96,6 +96,7 @@ TARGET_DIR TARGET_FILE_NAME TMP_CONTAINER_NAME TMP_SPACE_BUILD +USE_ADOPTIUM_DEVKIT USE_DOCKER USE_JEP319_CERTS USE_SSH @@ -119,6 +120,7 @@ WORKSPACE_DIR # /config Configuration /openjdk/config $(pwd)/workspace/config # / Build area /openjdk/build $(pwd)/workspace/build/ # // Source code /openjdk/build/src $(pwd)/workspace/build/src +# //devkit DevKit download /openjdk/build/devkit $(pwd)/workspace/build/devkit # /target Destination of built artifacts /openjdk/target $(pwd)/workspace/target # Helper code to perform index lookups by name @@ -353,6 +355,9 @@ function parseConfigurationArguments() { "--use-jep319-certs" ) BUILD_CONFIG[USE_JEP319_CERTS]=true;; + "--use-adoptium-devkit") + BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]="$1"; shift;; + "--user-openjdk-build-root-directory" ) BUILD_CONFIG[USER_OPENJDK_BUILD_ROOT_DIRECTORY]="$1"; shift;; @@ -597,6 +602,9 @@ function configDefaults() { BUILD_CONFIG[CLEAN_LIBS]=false + # Default to no Adoptium DevKit + BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]="" + # By default dont backport JEP318 certs to < Java 10 BUILD_CONFIG[USE_JEP319_CERTS]=false diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh index 67fc17cf1d..edc3a4e5fb 100644 --- a/sbin/prepareWorkspace.sh +++ b/sbin/prepareWorkspace.sh @@ -328,21 +328,8 @@ checkingAndDownloadingAlsa() { curl -o "alsa-lib.tar.bz2" "$ALSA_BUILD_URL" curl -o "alsa-lib.tar.bz2.sig" "https://www.alsa-project.org/files/pub/lib/alsa-lib-${ALSA_LIB_VERSION}.tar.bz2.sig" - ## This affects riscv64 & Alpine docker images and also evaluation pipelines - if ( [ -r /etc/alpine-release ] && [ "$(pwd | wc -c)" -gt 83 ] ) || \ - ( [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "linux" ] && [ "${BUILD_CONFIG[OS_ARCHITECTURE]}" == "riscv64" ] && [ "$(pwd | wc -c)" -gt 83 ] ); then - # Use /tmp in preference to $HOME as fails gpg operation if PWD > 83 characters - # Also cannot create ~/.gpg-temp within a docker context - GNUPGHOME="$(mktemp -d /tmp/.gpg-temp.XXXXXX)" - else - GNUPGHOME="${BUILD_CONFIG[WORKSPACE_DIR]:-$PWD}/.gpg-temp" - fi - if [ ! -d "$GNUPGHOME" ]; then - mkdir -m 700 "$GNUPGHOME" - fi - export GNUPGHOME + setupGpg - echo "GNUPGHOME=$GNUPGHOME" # Should we clear this directory up after checking? # Would this risk removing anyone's existing dir with that name? # Erring on the side of caution for now @@ -575,6 +562,67 @@ prepareMozillaCacerts() { fi } +# Create and setup GNUPGHOME +setupGpg() { + ## This affects riscv64 & Alpine docker images and also evaluation pipelines + if ( [ -r /etc/alpine-release ] && [ "$(pwd | wc -c)" -gt 83 ] ) || \ + ( [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "linux" ] && [ "${BUILD_CONFIG[OS_ARCHITECTURE]}" == "riscv64" ] && [ "$(pwd | wc -c)" -gt 83 ] ); then + # Use /tmp in preference to $HOME as fails gpg operation if PWD > 83 characters + # Also cannot create ~/.gpg-temp within a docker context + GNUPGHOME="$(mktemp -d /tmp/.gpg-temp.XXXXXX)" + else + GNUPGHOME="${BUILD_CONFIG[WORKSPACE_DIR]:-$PWD}/.gpg-temp" + fi + if [ ! -d "$GNUPGHOME" ]; then + mkdir -m 700 "$GNUPGHOME" + fi + export GNUPGHOME + + echo "GNUPGHOME=$GNUPGHOME" +} + +# Download the required DevKit if necessary +downloadDevkit() { + if [[ -n "${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" ]]; then + rm -rf "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit" + mkdir -p "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit" + + local devkit_tar="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit/devkit.tar.xz" + + setupGpg + + # Determine DevKit tarball to download for this arch and release + local devkitUrl="https://github.com/adoptium/devkit-binaries/releases/download/${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" + local devkit_target="${BUILD_CONFIG[OS_ARCHITECTURE]}-linux-gnu" + local devkit="devkit-${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}-${devkit_target}" + + # Download tarball and GPG sig + echo "Downloading DevKit : ${devkitUrl}/${devkit}.tar.xz" + curl -L --fail --silent --show-error -o "${devkit_tar}" "${devkitUrl}/${devkit}.tar.xz" + curl -L --fail --silent --show-error -o "${devkit_tar}.sig" "${devkitUrl}/${devkit}.tar.xz.sig" + + # GPG verify + gpg --keyserver keyserver.ubuntu.com --recv-keys 3B04D753C9050D9A5D343F39843C48A565F8F04B + echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key 3B04D753C9050D9A5D343F39843C48A565F8F04B trust; + gpg --verify "${devkit_tar}.sig" "${devkit_tar}" || exit 1 + + tar xpJf "${devkit_tar}" -C "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit" + rm "${devkit_tar}" + rm "${devkit_tar}.sig" + + # Validate devkit.info matches value passed in and current architecture + local devkitInfo="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit/devkit.info" + if ! grep "ADOPTIUM_DEVKIT_RELEASE=${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" "${devkitInfo}" || ! grep "ADOPTIUM_DEVKIT_TARGET=${devkit_target}" "${devkitInfo}"; then + echo "ERROR: Devkit does not match required release and architecture:" + echo " Required: ADOPTIUM_DEVKIT_RELEASE=${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" + echo " Downloaded: $(grep ADOPTIUM_DEVKIT_RELEASE= "${devkitInfo}")" + echo " Required: ADOPTIUM_DEVKIT_TARGET=${devkit_target}" + echo " Downloaded: $(grep ADOPTIUM_DEVKIT_TARGET= "${devkitInfo}")" + exit 1 + fi + fi +} + # Download all of the dependencies for OpenJDK (Alsa, FreeType etc.) downloadingRequiredDependencies() { if [[ "${BUILD_CONFIG[CLEAN_LIBS]}" == "true" ]]; then @@ -693,6 +741,7 @@ function configureWorkspace() { if [[ "${BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]}" != "true" ]]; then createWorkspace downloadingRequiredDependencies + downloadDevkit relocateToTmpIfNeeded checkoutAndCloneOpenJDKGitRepo applyPatches diff --git a/tooling/reproducible/windows_repro_build_compare.sh b/tooling/reproducible/windows_repro_build_compare.sh index 5d1aed039e..41ecc2c928 100644 --- a/tooling/reproducible/windows_repro_build_compare.sh +++ b/tooling/reproducible/windows_repro_build_compare.sh @@ -39,7 +39,8 @@ TARBALL_URL="$2" # These Values Should Be Updated To Reflect The Build Environment # The Defaults Below Are Suitable For An Adoptium Windows Build Environment # Which Has Been Created Via The Ansible Infrastructure Playbooks -WORK_DIR="/cmp$(date +%Y%m%d%H%M%S)" + +WORK_DIR="/cygdrive/c/comp-jdk-build" ANT_VERSION="1.10.5" ANT_CONTRIB_VERSION="1.0b3" ANT_BASE_PATH="/cygdrive/c/apache-ant" @@ -55,12 +56,13 @@ SIGNTOOL_BASE="C:/Program Files (x86)/Windows Kits/10" # Define What Are Configure Args & Redundant Args # This MAY Need Updating If Additional Configure Args Are Passed -CONFIG_ARGS=("--disable-warnings-as-errors" "--disable-ccache" "--with-toolchain-version" "--with-ucrt-dll-dir") +CONFIG_ARGS=("--disable-warnings-as-errors" "--disable-ccache" "--with-toolchain-version" "--with-ucrt-dll-dir" "--with-version-opt") NOTUSE_ARGS=("--assemble-exploded-image" "--configure-args") # Addiitonal Working Variables Defined For Use By This Script SBOMLocalPath="$WORK_DIR/src_sbom.json" DISTLocalPath="$WORK_DIR/src_jdk_dist.zip" +ScriptPath="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Function to check if a string is a valid URL is_url() { @@ -92,7 +94,7 @@ Create_WorkDir() { echo "Folder Exists - Removing '$WORK_DIR'" fi echo "Creating $WORK_DIR" - mkdir "$WORK_DIR" + mkdir -p "$WORK_DIR" } # Function To Check The SBOM @@ -138,37 +140,41 @@ Check_Parameters() { } Install_PreReqs() { - # Install Cygwin Package Manager If Not Present - echo "Checking If Apt-Cyg Is Already Installed" - if [ -f /usr/local/bin/apt-cyg ]; then - echo "Skipping apt-cyg Install" - APTCYG_INSTALLED="True" - else - echo "Installing apt-cyg" - APTCYG_INSTALLED="False" - wget -q -O "./apt-cyg" "https://raw.githubusercontent.com/transcode-open/apt-cyg/master/apt-cyg" - ACTSHASUM=$(sha256sum "apt-cyg" | awk '{print $1}') - EXPSHASUM="d020050e2cb56fec990f16fd10695e153afd064cb0839ba935247b5a9e4c29a0" - - if [ "$ACTSHASUM" == "$EXPSHASUM" ]; then - chmod +x apt-cyg - mv apt-cyg /usr/local/bin - else - echo "Checksum Is Not OK - Exiting" - exit 1 - fi + # Check For JQ & Install Apt-Cyg & JQ Where Not Available + if ! command -v jq &> /dev/null; then + echo "WARNING: JQ is not installed. Attempting To Install Via Apt-Cyg" + echo "Checking If Apt-Cyg Is Already Installed" + if [ -f /usr/local/bin/apt-cyg ]; then + echo "Skipping apt-cyg Install" + APTCYG_INSTALLED="True" + else + echo "Installing apt-cyg" + APTCYG_INSTALLED="False" + wget -q -O "./apt-cyg" "https://raw.githubusercontent.com/transcode-open/apt-cyg/master/apt-cyg" + ACTSHASUM=$(sha256sum "apt-cyg" | awk '{print $1}') + EXPSHASUM="d020050e2cb56fec990f16fd10695e153afd064cb0839ba935247b5a9e4c29a0" + if [ "$ACTSHASUM" == "$EXPSHASUM" ]; then + chmod +x apt-cyg + mv apt-cyg /usr/local/bin + else + echo "Checksum Is Not OK - Exiting" + exit 1 + fi + fi + + echo "Checking If JQ Is Already Installed" + if [ -f /usr/local/bin/jq ]; then + echo "Skipping JQ Install" + APTJQ_INSTALLED="True" + else + echo "Installing JQ via APTCYG" + APTJQ_INSTALLED="False" + apt-cyg install jq libjq1 libonig5 + fi fi # Install JQ Where Not Already Installed - echo "Checking If JQ Is Already Installed" - if [ -f /usr/local/bin/jq ]; then - echo "Skipping JQ Install" - APTJQ_INSTALLED="True" - else - echo "Installing JQ via APTCYG" - APTJQ_INSTALLED="False" - apt-cyg install jq libjq1 libonig5 - fi + } Get_SBOM_Values() { @@ -281,7 +287,7 @@ Get_SBOM_Values() { echo "Temurin Build Arguments: $buildArgs" export buildArgs else - echo "ERROR: Temurin Build Version not found in the SBOM." + echo "ERROR: Temurin Build Arguments not found in the SBOM." echo "This Is A Mandatory Element" exit 1 fi @@ -567,7 +573,7 @@ Prepare_Env_For_Build() { # Loop through the words for word in "${words[@]}"; do # Check if the word starts with '--' - if [[ $word == --* ]]; then + if [[ $word == --* ]] || [[ $word == -b* ]]; then # If a parameter already exists, store it in the params array if [[ -n $param ]]; then params+=("$param=$value") @@ -580,7 +586,7 @@ Prepare_Env_For_Build() { fi done - # Add the last parameter to the array + # Add the last parameter to the array params+=("$param = $value") # Read the separated parameters and values into a new array @@ -600,13 +606,15 @@ Prepare_Env_For_Build() { fixed_value=$(echo "$prepped_value" | awk '{$1=$1};1') # Handle Special parameters + if [ "$fixed_param" == "-b" ]; then fixed_value="$fixed_value " ; fi if [ "$fixed_param" == "--jdk-boot-dir" ]; then fixed_value="$BOOTJDK_HOME " ; fi if [ "$fixed_param" == "--freetype-dir" ]; then fixed_value="$fixed_value " ; fi if [ "$fixed_param" == "--with-toolchain-version" ]; then fixed_value="$visualStudioVersion " ; fi - if [ "$fixed_param" == "--with-ucrt-dll-dir" ]; then fixed_value="temporary_speech_mark_placeholder${UCRT_PARAM_PATH}temporary_speech_mark_placeholder" ; fi + if [ "$fixed_param" == "--with-ucrt-dll-dir" ]; then fixed_value="temporary_speech_mark_placeholder${UCRT_PARAM_PATH}temporary_speech_mark_placeholder " ; fi if [ "$fixed_param" == "--target-file-name" ]; then target_file="$fixed_value" ; fixed_value="$fixed_value " ; fi if [ "$fixed_param" == "--tag" ]; then fixed_value="$fixed_value " ; fi + # Fix Build Variant Parameter To Strip JDK Version if [ "$fixed_param" == "--build-variant" ] ; then @@ -628,7 +636,7 @@ Prepare_Env_For_Build() { # Add Config Arg To New Array # Handle Windows Param Names In Config Args (Replace Space with =) - if [ "$fixed_param" == "--with-toolchain-version" ] || [ "$fixed_param" == "--with-ucrt-dll-dir" ] ; then + if [ "$fixed_param" == "--with-toolchain-version" ] || [ "$fixed_param" == "--with-ucrt-dll-dir" ] || [ "$fixed_param" == "--with-version-opt" ] ; then STRINGTOADD="$fixed_param=$fixed_value" CONFIG_ARRAY+=("$STRINGTOADD") else @@ -675,7 +683,6 @@ Build_JDK() { Compare_JDK() { echo "Comparing JDKs" echo "" - cd "$WORK_DIR" mkdir "$WORK_DIR/compare" cp "$WORK_DIR/src_jdk_dist.zip" "$WORK_DIR/compare" cp "$WORK_DIR/built_jdk.zip" "$WORK_DIR/compare" @@ -749,15 +756,12 @@ Compare_JDK() { export JAVA_HOME=$BOOTJDK_HOME export PATH=$JAVA_HOME/bin:$PATH - # Run Comparison Script - cd "$WORK_DIR/compare" - CPW=$(cygpath -u "$SIGNPATHWIN") export PATH="$PATH:$CPW" # Run Comparison Script - cd "$WORK_DIR/compare" - ./repro_compare.sh temurin src_jdk temurin tar_jdk CYGWIN + echo "cd $WORK_DIR/compare && ./repro_compare.sh temurin src_jdk temurin tar_jdk CYGWIN 2>&1" | sh & + wait # Display The Content Of repro_diff.out echo "" @@ -767,7 +771,9 @@ Compare_JDK() { cat "$WORK_DIR/compare/repro_diff.out" echo "" echo "---------------------------------------------" - mv "$WORK_DIR/compare/repro_diff.out" "$WORK_DIR" + echo "Copying Output To $(dirname "$0")" + cp "$WORK_DIR/compare/repro_diff.out" "$ScriptPath" + } Clean_Up_Everything() {