From e7ffbd122455347314b4b80edec8ec9b707476fb Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Wed, 15 Nov 2023 10:30:06 +0000 Subject: [PATCH 01/11] Add Windows comparable patch fixes Signed-off-by: U-andrew-repro-te\adoptium --- tooling/comparable_patch.sh | 223 +++++++++++++++++++++++++++++------- 1 file changed, 184 insertions(+), 39 deletions(-) diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh index 88cef75ba..d2450cee5 100755 --- a/tooling/comparable_patch.sh +++ b/tooling/comparable_patch.sh @@ -28,6 +28,11 @@ set -eu TEMURIN_TOOLS_BINREPL="temurin.tools.BinRepl" +if [ "$#" -ne 6 ]; then + echo "Syntax: comparable_patch.sh " + exit 1 +fi + JDK_DIR="$1" VERSION_REPL="$2" VENDOR_NAME="$3" @@ -37,11 +42,8 @@ VENDOR_VM_BUG_URL="$6" # Remove excluded files known to differ function removeExcludedFiles() { - if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class" - else - excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa" - fi + excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa" + echo "Removing excluded files known to differ: ${excluded}" for exclude in $excluded do @@ -53,12 +55,141 @@ function removeExcludedFiles() { done done + echo "Successfully removed all excluded files from ${JDK_DIR}" +} + +# Normalize the following ModuleAttributes that can be ordered differently +# depending on how the vendor has signed and re-packed the JMODs +# - ModuleResolution: +# - ModuleTarget: +# java.base also requires the dependent module "hash:" values to be excluded +# as they differ due to the Signatures +function processModuleInfo() { if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - echo "Removing java.base module-info.class, known to differ by jdk.jpackage module hash" - rm "${JDK_DIR}/jmods/expanded_java.base.jmod/classes/module-info.class" - rm "${JDK_DIR}/lib/modules_extracted/java.base/module-info.class" + echo "Normalizing ModuleAttributes order in module-info.class, converting to javap.tmp" + + moduleAttr="ModuleResolution ModuleTarget" + + FILES=$(find "${JDK_DIR}" -type f -name "module-info.class") + for f in $FILES + do + echo "javap and re-order ModuleAttributes for $f" + javap -v -sysinfo -l -p -c -s -constants "$f" > "$f.javap.tmp" + rm "$f" + + cc=99 + foundAttr=false + attrName="" + # Clear any attr tmp files + for attr in $moduleAttr + do + rm -f "$f.javap.$attr" + done + + while IFS= read -r line + do + cc=$((cc+1)) + + # Module attr have only 1 line definition + if [[ "$foundAttr" = true ]] && [[ "$cc" > 1 ]]; then + foundAttr=false + attrName="" + fi + + # If not processing an attr then check for attr + if [[ "$foundAttr" = false ]]; then + for attr in $moduleAttr + do + if [[ "$line" =~ .*"$attr:".* ]]; then + cc=0 + foundAttr=true + attrName="$attr" + fi + done + fi + + # Echo attr to attr tmp file, otherwise to tmp2 + if [[ "$foundAttr" = true ]]; then + echo "$line" >> "$f.javap.$attrName" + else + echo "$line" >> "$f.javap.tmp2" + fi + done < "$f.javap.tmp" + rm "$f.javap.tmp" + + # Remove javap Classfile and timestamp and SHA-256 hash + if [[ "$f" =~ .*"java.base".* ]]; then + grep -v "Last modified\|Classfile\|SHA-256 checksum\|hash:" "$f.javap.tmp2" > "$f.javap" + else + grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" + fi + rm "$f.javap.tmp2" + + # Append any ModuleAttr tmp files + for attr in $moduleAttr + do + if [[ -f "$f.javap.$attr" ]]; then + cat "$f.javap.$attr" >> "$f.javap" + fi + rm -f "$f.javap.$attr" + done + done fi - echo "Successfully removed all excluded files from ${JDK_DIR}" +} + +# Process SystemModules classes to remove ModuleHashes$Builder differences due to Signatures +# 1. javap +# 2. search for line: // Method jdk/internal/module/ModuleHashes$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes$Builder; +# 3. followed 3 lines later by: // String +# 4. then remove all lines until next: invokevirtual +# 5. remove Last modified, Classfile and SHA-256 checksum javap artefact statements +function removeSystemModulesHashBuilderParams() { + # Key strings + moduleHashesFunction="// Method jdk/internal/module/ModuleHashes\$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes\$Builder;" + moduleString="// String " + virtualFunction="invokevirtual" + + systemModules="SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class" + echo "Removing SystemModules ModulesHashes\$Builder differences" + for systemModule in $systemModules + do + FILES=$(find "${JDK_DIR}" -type f -name "$systemModule") + for f in $FILES + do + echo "Processing $f" + javap -v -sysinfo -l -p -c -s -constants "$f" > "$f.javap.tmp" + rm "$f" + + # Remove "instruction number:" prefix, so we can just match code + sed -i -E "s/^[[:space:]]+[0-9]+:(.*)/\1/" "$f.javap.tmp" + + cc=99 + found=false + while IFS= read -r line + do + cc=$((cc+1)) + if [[ "$line" =~ .*"$moduleHashesFunction".* ]]; then + cc=0 + fi + if [[ "$cc" == 3 ]] && [[ "$line" =~ .*"$moduleString"[a-z\.]+.* ]]; then + found=true + module=$(echo "$line" | tr -s ' ' | tr -d '\r' | cut -d' ' -f6) + echo "==> Found $module ModuleHashes\$Builder function, skipping hash parameter" + fi + if [[ "$found" = true ]] && [[ "$line" =~ .*"$virtualFunction".* ]]; then + found=false + fi + if [[ "$found" = false ]]; then + echo "$line" >> "$f.javap.tmp2" + fi + done < "$f.javap.tmp" + rm "$f.javap.tmp" + grep -v "Last modified\|Classfile\|SHA-256 checksum" "$f.javap.tmp2" > "$f.javap" + rm "$f.javap.tmp2" + done + done + + echo "Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}" } # Remove the Windows EXE/DLL timestamps and internal VS CRC and debug repro hex values @@ -68,15 +199,20 @@ function removeWindowsNonComparableData() { for f in $FILES do echo "Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f" - rm -f dumpbin.tmp - if ! dumpbin "$f" /ALL > dumpbin.tmp; then - echo " FAILED == > dumpbin \"$f\" /ALL > dumpbin.tmp" + + # Determine non-comparable data using dumpbin + dmpfile="$f.dumpbin.tmp" + rm -f "$dmpfile" + if ! dumpbin "$f" /ALL > "$dmpfile"; then + echo " FAILED == > dumpbin \"$f\" /ALL > $dmpfile" exit 1 fi - timestamp=$(grep "time date stamp" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f2) - checksum=$(grep "checksum" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f2) - reprohex=$(grep "${timestamp} repro" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r') - reprohexhalf=$(grep "${timestamp} repro" dumpbin.tmp | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r') + timestamp=$(grep "time date stamp" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2) + checksum=$(grep "checksum" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2) + reprohex=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r') + reprohexhalf=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r') + rm -f "$dmpfile" + if [ -n "$reprohex" ]; then if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA"; then echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\"" @@ -231,39 +367,44 @@ function neutraliseReleaseFile() { echo "Removing Vendor strings from release file ${JDK_DIR}/release" if [[ "$OS" =~ Darwin* ]]; then + # Remove Vendor versions sed -i "" "s=$VERSION_REPL==g" "${JDK_DIR}/release" sed -i "" "s=$VENDOR_NAME==g" "${JDK_DIR}/release" - # BUILD_INFO likely different since built on different machines - sed -i "" "s=^BUILD_INFO.*$==g" "${JDK_DIR}/release" - - # BUILD_SOURCE possibly built not using temurin build scripts - sed -i "" "s=^BUILD_SOURCE.*$==g" "${JDK_DIR}/release" - sed -i "" "s=^BUILD_SOURCE_REPO.*$==g" "${JDK_DIR}/release" - - # Temurin JVM_VARIANT has capital "Hotspot" - sed -i "" "s,^JVM_VARIANT=\"Hotspot\",JVM_VARIANT=\"hotspot\",g" "${JDK_DIR}/release" + # Temurin BUILD_* likely different since built on different machines and bespoke to Temurin + sed -i "" "/^BUILD_INFO/d" "${JDK_DIR}/release" + sed -i "" "/^BUILD_SOURCE/d" "${JDK_DIR}/release" + sed -i "" "/^BUILD_SOURCE_REPO/d" "${JDK_DIR}/release" + + # Remove bespoke Temurin fields + sed -i "" "/^SOURCE/d" "${JDK_DIR}/release" + sed -i "" "/^FULL_VERSION/d" "${JDK_DIR}/release" + sed -i "" "/^SEMANTIC_VERSION/d" "${JDK_DIR}/release" + sed -i "" "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "" "/^JVM_VERSION/d" "${JDK_DIR}/release" + sed -i "" "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "" "/^IMAGE_TYPE/d" "${JDK_DIR}/release" else + # Remove Vendor versions sed -i "s=$VERSION_REPL==g" "${JDK_DIR}/release" sed -i "s=$VENDOR_NAME==g" "${JDK_DIR}/release" - # BUILD_INFO likely different since built on different machines - sed -i "s=^BUILD_INFO.*$==g" "${JDK_DIR}/release" - - # BUILD_SOURCE possibly built not using temurin build scripts - sed -i "s=^BUILD_SOURCE.*$==g" "${JDK_DIR}/release" - sed -i "s=^BUILD_SOURCE_REPO.*$==g" "${JDK_DIR}/release" - - # Temurin JVM_VARIANT has capital "Hotspot" - sed -i "s,^JVM_VARIANT=\"Hotspot\",JVM_VARIANT=\"hotspot\",g" "${JDK_DIR}/release" + # Temurin BUILD_* likely different since built on different machines and bespoke to Temurin + sed -i "/^BUILD_INFO/d" "${JDK_DIR}/release" + sed -i "/^BUILD_SOURCE/d" "${JDK_DIR}/release" + sed -i "/^BUILD_SOURCE_REPO/d" "${JDK_DIR}/release" + + # Remove bespoke Temurin fields + sed -i "/^SOURCE/d" "${JDK_DIR}/release" + sed -i "/^FULL_VERSION/d" "${JDK_DIR}/release" + sed -i "/^SEMANTIC_VERSION/d" "${JDK_DIR}/release" + sed -i "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "/^JVM_VERSION/d" "${JDK_DIR}/release" + sed -i "/^JVM_VARIANT/d" "${JDK_DIR}/release" + sed -i "/^IMAGE_TYPE/d" "${JDK_DIR}/release" fi } -if [ "$#" -ne 6 ]; then - echo "Syntax: cmd " - exit 1 -fi - if [ ! -d "${JDK_DIR}" ]; then echo "$JDK_DIR does not exist" exit 1 @@ -299,6 +440,10 @@ echo "Successfully removed all Signatures from ${JDK_DIR}" removeExcludedFiles +processModuleInfo + +removeSystemModulesHashBuilderParams + if [[ "$OS" =~ CYGWIN* ]]; then neutraliseVsVersionInfo fi From d56340bbc22ea9100f5fcefc421a2321b7e722dc Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Wed, 15 Nov 2023 10:41:25 +0000 Subject: [PATCH 02/11] Add Windows comparable patch fixes Signed-off-by: U-andrew-repro-te\adoptium --- tooling/comparable_patch.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh index d2450cee5..b172e0245 100755 --- a/tooling/comparable_patch.sh +++ b/tooling/comparable_patch.sh @@ -405,6 +405,16 @@ function neutraliseReleaseFile() { fi } +# Remove some non-JDK files that some Vendors distribute +# - NEWS : Some Vendors provide a NEWS text file +# - demo : Not all vendors distribute the demo examples +function removeNonJdkFiles() { + echo "Removing non-JDK files" + + rm -f "${JDK_DIR}/NEWS" + rm -rf "${JDK_DIR}/demo" +} + if [ ! -d "${JDK_DIR}" ]; then echo "$JDK_DIR does not exist" exit 1 @@ -464,6 +474,8 @@ neutraliseManifests neutraliseReleaseFile +removeNonJdkFiles + echo "***********" echo "SUCCESS :-)" echo "***********" From 73880f2eebb040966497315f5ed9d980e4df176a Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Wed, 15 Nov 2023 14:40:56 +0000 Subject: [PATCH 03/11] Add Windows comparable patch fixes Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 25 ++++++++++++++++++++++++- tooling/comparable_patch.sh | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index 6f565da6a..c3929be06 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -39,7 +39,7 @@ remove any Signatures from the executeables. ## Comparable Build Tools -1. comparable_patch.sh : Patches a JDK folder to enable "Comparable" comparison of two different Vendor JDK builds. +comparable_patch.sh : Patches a JDK folder to enable "Comparable" comparison of two different Vendor JDK builds. The patching process involves: - Expanding all zips and jmods, so executeables can be processed to remove signatures prior to comaprison. @@ -47,4 +47,27 @@ The patching process involves: - Neutralise VS_VERSION_INFO Vendor strings (Windows). - Remove non-comparable CRC generated uuid values, which are binary values based on the hash of the content (Windows & MacOS). - Remove Vendor strings embedded in executables, classes and text files. +- Remove module-info differences due to "hash" of Signed module executables - Remove any non-deterministic build process artifact strings, like Manifest Created-By stamps. + +### How to setup and run comparable_patch.sh on Windows + +Tooling setup: + +1. The comparable patch tools, tooling/src/c/WindowsUpdateVsVersionInfo.c and src/java/temurin/tools/BinRepl.java need compiling +before the comparable_patch.sh can be run +2. Compile tooling/src/c/WindowsUpdateVsVersionInfo.c : + - Ensure VS2022 SDK is installed and on PATH + - Compile: + - cd tooling/src/c + - cl WindowsUpdateVsVersionInfo.c version.lib +3. Compile src/java/temurin/tools/BinRepl.java : + - Ensure suitable JDK on PATH + - cd tooling/src/java + - javac temurin/tools/BinRepl.java +4. Setting environment within a CYGWIN shell : + - For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH + - For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH + - For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra~1/micros~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH + - For running BinRepl java : export PATH=/bin:$PATH + diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh index b172e0245..39f5ee5e5 100755 --- a/tooling/comparable_patch.sh +++ b/tooling/comparable_patch.sh @@ -66,7 +66,7 @@ function removeExcludedFiles() { # as they differ due to the Signatures function processModuleInfo() { if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then - echo "Normalizing ModuleAttributes order in module-info.class, converting to javap.tmp" + echo "Normalizing ModuleAttributes order in module-info.class, converting to javap" moduleAttr="ModuleResolution ModuleTarget" From 8ae7f57908da8acbad699cc5e83954ed40858a4e Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Thu, 16 Nov 2023 10:16:23 +0000 Subject: [PATCH 04/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 26 ++++++++++++++------------ tooling/comparable_patch.sh | 23 ++++++++++++++++++++--- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index c3929be06..ea6504b73 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -56,18 +56,20 @@ Tooling setup: 1. The comparable patch tools, tooling/src/c/WindowsUpdateVsVersionInfo.c and src/java/temurin/tools/BinRepl.java need compiling before the comparable_patch.sh can be run + 2. Compile tooling/src/c/WindowsUpdateVsVersionInfo.c : - - Ensure VS2022 SDK is installed and on PATH - - Compile: - - cd tooling/src/c - - cl WindowsUpdateVsVersionInfo.c version.lib +- Ensure VS2022 SDK is installed and on PATH +- Compile: + - cd tooling/src/c + - cl WindowsUpdateVsVersionInfo.c version.lib + 3. Compile src/java/temurin/tools/BinRepl.java : - - Ensure suitable JDK on PATH - - cd tooling/src/java - - javac temurin/tools/BinRepl.java -4. Setting environment within a CYGWIN shell : - - For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH - - For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH - - For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra~1/micros~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH - - For running BinRepl java : export PATH=/bin:$PATH +- Ensure suitable JDK on PATH +- cd tooling/src/java +- javac temurin/tools/BinRepl.java +4. Setting environment within a CYGWIN shell : +- For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH +- For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH +- For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra~1/micros~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH +- For running BinRepl java : export PATH=/bin:$PATH diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh index 39f5ee5e5..40f54e93c 100755 --- a/tooling/comparable_patch.sh +++ b/tooling/comparable_patch.sh @@ -41,6 +41,9 @@ VENDOR_BUG_URL="$5" VENDOR_VM_BUG_URL="$6" # Remove excluded files known to differ +# NOTICE - Vendor specfic notice text file +# cacerts - Vendors use different cacerts +# classes.jsa, classes_nocoops.jsa - CDS archive caches will differ due to Vendor string differences function removeExcludedFiles() { excluded="NOTICE cacerts classes.jsa classes_nocoops.jsa" @@ -91,7 +94,7 @@ function processModuleInfo() { cc=$((cc+1)) # Module attr have only 1 line definition - if [[ "$foundAttr" = true ]] && [[ "$cc" > 1 ]]; then + if [[ "$foundAttr" = true ]] && [[ "$cc" -gt 1 ]]; then foundAttr=false attrName="" fi @@ -168,14 +171,17 @@ function removeSystemModulesHashBuilderParams() { while IFS= read -r line do cc=$((cc+1)) + # Detect hashForModule function if [[ "$line" =~ .*"$moduleHashesFunction".* ]]; then cc=0 fi - if [[ "$cc" == 3 ]] && [[ "$line" =~ .*"$moduleString"[a-z\.]+.* ]]; then + # 3rd instruction line is the Module string to confirm entry + if [[ "$cc" -eq 3 ]] && [[ "$line" =~ .*"$moduleString"[a-z\.]+.* ]]; then found=true module=$(echo "$line" | tr -s ' ' | tr -d '\r' | cut -d' ' -f6) echo "==> Found $module ModuleHashes\$Builder function, skipping hash parameter" fi + # hasForModule function section finishes upon finding invokevirtual if [[ "$found" = true ]] && [[ "$line" =~ .*"$virtualFunction".* ]]; then found=false fi @@ -193,6 +199,11 @@ function removeSystemModulesHashBuilderParams() { } # Remove the Windows EXE/DLL timestamps and internal VS CRC and debug repro hex values +# The Windows PE format contains various values determined from the binary content +# which will vary due to the different Vendor branding +# timestamp - Used to be an actual timestamp but MSFT changed this to a checksum determined from binary content +# checksum - A checksum value of the binary +# reprohex - A hex UUID to identify the binary version, again generated from binary content function removeWindowsNonComparableData() { echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}" FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll') @@ -207,18 +218,23 @@ function removeWindowsNonComparableData() { echo " FAILED == > dumpbin \"$f\" /ALL > $dmpfile" exit 1 fi + + # Determine non-comparable stamps and hex codes from dumpbin output timestamp=$(grep "time date stamp" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2) checksum=$(grep "checksum" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f2) reprohex=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-38 | tr ' ' ':' | tr -d '\r') reprohexhalf=$(grep "${timestamp} repro" "$dmpfile" | head -1 | tr -s ' ' | cut -d' ' -f7-22 | tr ' ' ':' | tr -d '\r') rm -f "$dmpfile" + # Neutralize reprohex string if [ -n "$reprohex" ]; then if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA"; then echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${reprohex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\"" exit 1 fi fi + + # Neutralize timestamp hex string hexstr="00000000" timestamphex=${hexstr:0:-${#timestamp}}$timestamp timestamphexLE="${timestamphex:6:2}:${timestamphex:4:2}:${timestamphex:2:2}:${timestamphex:0:2}" @@ -233,6 +249,7 @@ function removeWindowsNonComparableData() { fi fi + # Neutralize checksum string # Prefix checksum to 8 digits hexstr="00000000" checksumhex=${hexstr:0:-${#checksum}}$checksum @@ -272,7 +289,7 @@ function removeMacOSNonComparableData() { echo "Successfully removed all MacOS dylib non-comparable UUID from ${JDK_DIR}" } -# Neutralize Windows VS_VERSION_INFO CompanyName +# Neutralize Windows VS_VERSION_INFO CompanyName from the resource compiled PE section function neutraliseVsVersionInfo() { echo "Updating EXE/DLL VS_VERSION_INFO in ${JDK_DIR}" FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll') From cbd9d33e492512d3f263c424276e403db324c584 Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Thu, 16 Nov 2023 10:22:41 +0000 Subject: [PATCH 05/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index ea6504b73..8d1e798d5 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -58,17 +58,20 @@ Tooling setup: before the comparable_patch.sh can be run 2. Compile tooling/src/c/WindowsUpdateVsVersionInfo.c : + - Ensure VS2022 SDK is installed and on PATH - Compile: - cd tooling/src/c - cl WindowsUpdateVsVersionInfo.c version.lib 3. Compile src/java/temurin/tools/BinRepl.java : + - Ensure suitable JDK on PATH - cd tooling/src/java - javac temurin/tools/BinRepl.java 4. Setting environment within a CYGWIN shell : + - For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH - For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH - For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra~1/micros~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH From 3e3b748e37605d8c9d4f923764f95aa9aa53fd8b Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Thu, 16 Nov 2023 10:25:22 +0000 Subject: [PATCH 06/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index 8d1e798d5..92e952d5c 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -74,5 +74,5 @@ before the comparable_patch.sh can be run - For BinRepl : export CLASSPATH=/tooling/src/java:$CLASSPATH - For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH -- For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra~1/micros~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH +- For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra\~1/micros\~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH - For running BinRepl java : export PATH=/bin:$PATH From 33b2ebddbc29cc6b30817d92eaea71c574dd4529 Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Fri, 17 Nov 2023 09:13:05 +0000 Subject: [PATCH 07/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 36 ++++++++++++++++++++++++++++++++++- tooling/comparable_patch.sh | 2 +- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index 92e952d5c..9f976dac7 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -52,7 +52,7 @@ The patching process involves: ### How to setup and run comparable_patch.sh on Windows -Tooling setup: +#### Tooling setup: 1. The comparable patch tools, tooling/src/c/WindowsUpdateVsVersionInfo.c and src/java/temurin/tools/BinRepl.java need compiling before the comparable_patch.sh can be run @@ -76,3 +76,37 @@ before the comparable_patch.sh can be run - For WindowsUpdateVsVersionInfo.exe : export PATH=/tooling/src/c:$PATH - For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra\~1/micros\~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH - For running BinRepl java : export PATH=/bin:$PATH + +#### Running comparable_patch.sh: + +1. Unzip your JDK archive into a directory (eg.jdk1) + +2. Run comparable_patch.sh + +``` +bash comparable_patch.sh +``` +The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": +``` +java -XswhoSettings: +... + java.vendor = Eclipse Adoptium + java.vendor.url = https://adoptium.net/ + java.vendor.url.bug = https://github.com/adoptium/adoptium-support/issues + java.vendor.version = Temurin-21.0.1+12 +... +``` +eg. +``` +bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues" +``` + +3. Unzip the other Vendor JDK to compare with, say into "jdk2", and run similar comparable_patch.sh +for that Vendor branding + +4. Diff recursively the now Vendor neutralized jdk directories + +``` +diff -r jdk1 jdk2 +``` +The diff should be "identical" if the two Vendor JDK's are "Comparable", ie."Identical except for the Vendor Branding" diff --git a/tooling/comparable_patch.sh b/tooling/comparable_patch.sh index 40f54e93c..52003b1ba 100755 --- a/tooling/comparable_patch.sh +++ b/tooling/comparable_patch.sh @@ -29,7 +29,7 @@ set -eu TEMURIN_TOOLS_BINREPL="temurin.tools.BinRepl" if [ "$#" -ne 6 ]; then - echo "Syntax: comparable_patch.sh " + echo "Syntax: comparable_patch.sh " exit 1 fi From 4932d002bb0aa08a2ae3b4f592276bc31239871b Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Fri, 17 Nov 2023 09:16:03 +0000 Subject: [PATCH 08/11] Update doc Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index 9f976dac7..ab053ce8a 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -88,7 +88,7 @@ bash comparable_patch.sh ``` The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": ``` -java -XswhoSettings: +java -XshowSettings: ... java.vendor = Eclipse Adoptium java.vendor.url = https://adoptium.net/ @@ -101,7 +101,7 @@ eg. bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues" ``` -3. Unzip the other Vendor JDK to compare with, say into "jdk2", and run similar comparable_patch.sh +3. Unzip the other Vendor JDK to compare with, say into "jdk2", and run a similar comparable_patch.sh for that Vendor branding 4. Diff recursively the now Vendor neutralized jdk directories From f4eefe841b9786be18f42ab434c1e1b75a5e17a6 Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Fri, 17 Nov 2023 09:26:57 +0000 Subject: [PATCH 09/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index ab053ce8a..b07aee3cc 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -77,16 +77,21 @@ before the comparable_patch.sh can be run - For dumpbin.exe MSVC tool : export PATH=/cygdrive/c/progra\~1/micros\~2/2022/Community/VC/Tools/MSVC/14.37.32822/bin/Hostx64/x64:$PATH - For running BinRepl java : export PATH=/bin:$PATH -#### Running comparable_patch.sh: +#### Running comparable_patch.sh: 1. Unzip your JDK archive into a directory (eg.jdk1) 2. Run comparable_patch.sh + ``` bash comparable_patch.sh + ``` + The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": + + ``` java -XshowSettings: ... @@ -96,7 +101,9 @@ java -XshowSettings: java.vendor.version = Temurin-21.0.1+12 ... ``` + eg. + ``` bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues" ``` @@ -106,7 +113,9 @@ for that Vendor branding 4. Diff recursively the now Vendor neutralized jdk directories + ``` diff -r jdk1 jdk2 ``` + The diff should be "identical" if the two Vendor JDK's are "Comparable", ie."Identical except for the Vendor Branding" From 528e4d366bb8d6c8608062a4b7a24ae4005b0c88 Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Fri, 17 Nov 2023 09:31:42 +0000 Subject: [PATCH 10/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index b07aee3cc..638a9036d 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -83,16 +83,14 @@ before the comparable_patch.sh can be run 2. Run comparable_patch.sh - -``` +bash:``` bash comparable_patch.sh ``` The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": - -``` +java:``` java -XshowSettings: ... java.vendor = Eclipse Adoptium @@ -104,7 +102,7 @@ java -XshowSettings: eg. -``` +bash:``` bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues" ``` @@ -113,8 +111,7 @@ for that Vendor branding 4. Diff recursively the now Vendor neutralized jdk directories - -``` +bash:``` diff -r jdk1 jdk2 ``` From 502fe782258bff0afb3840c7ddc5dfc338c36e5b Mon Sep 17 00:00:00 2001 From: "U-andrew-repro-te\\adoptium" Date: Fri, 17 Nov 2023 09:40:40 +0000 Subject: [PATCH 11/11] Fix linter Signed-off-by: U-andrew-repro-te\adoptium --- tooling/ReproducibleBuilds.MD | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tooling/ReproducibleBuilds.MD b/tooling/ReproducibleBuilds.MD index 638a9036d..bf930e86d 100644 --- a/tooling/ReproducibleBuilds.MD +++ b/tooling/ReproducibleBuilds.MD @@ -83,14 +83,13 @@ before the comparable_patch.sh can be run 2. Run comparable_patch.sh -bash:``` +```bash bash comparable_patch.sh - ``` The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": -java:``` +```java java -XshowSettings: ... java.vendor = Eclipse Adoptium @@ -102,7 +101,7 @@ java -XshowSettings: eg. -bash:``` +```bash bash comparable_patch.sh jdk1/jdk-21.0.1+12 "Temurin-21.0.1+12" "Eclipse Adoptium" "https://adoptium.net/" "https://github.com/adoptium/adoptium-support/issues" "https://github.com/adoptium/adoptium-support/issues" ``` @@ -111,7 +110,7 @@ for that Vendor branding 4. Diff recursively the now Vendor neutralized jdk directories -bash:``` +```bash diff -r jdk1 jdk2 ```