From da293665a442e50c9d5d8bf8fcd1b9a908d3e9ab Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 26 Apr 2024 18:42:15 +0200 Subject: [PATCH 01/30] added podman to switches with -D podman is preffered, becuase if there is podman, then docker woudl be podman wrapper, and it is not 1:1 with original docker All other work shoudl be in signalhandler.sh and docker-build.sh and not sure how with ./docker/buildDocker.sh --- README.md | 8 ++++---- docker-build.sh | 2 ++ makejdk-any-platform.1 | 6 +++--- makejdk-any-platform.sh | 4 ++-- sbin/build.sh | 2 +- sbin/common/config_init.sh | 10 ++++++++-- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f9ced9753..11f38584a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ as we can generate valid dockerfile for it): ```bash ./makejdk-any-platform.sh --docker --clean-docker-build jdk8u -./makejdk-any-platform.sh --docker --clean-docker-build --build-variant openj9 jdk11u +./makejdk-any-platform.sh --podman --clean-docker-build --build-variant openj9 jdk11u ``` We test these dockerfiles on a regular basis in the @@ -144,8 +144,8 @@ specify the location for the built binary, e.g. /path/. This is typically used in conjunction with -T to create a custom path / file name for the resulting binary. --D, --docker -build OpenJDK in a docker container. +-D, --docker, --podman +build OpenJDK in a docker/podman container. -D will autodetect, using podman if found, docker otherwise. --cross-compile use this if you are cross compiling - it will skip the java -version checks at the end @@ -253,7 +253,7 @@ specify the JVM variant (server or client), defaults to server. Example usage: -./makejdk-any-platform --docker jdk8u +./makejdk-any-platform -D jdk8u ./makejdk-any-platform -T MyOpenJDK10.tar.gz jdk10 ``` diff --git a/docker-build.sh b/docker-build.sh index a6bbb80e0..62a46528d 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -71,6 +71,8 @@ buildDockerContainer() buildOpenJDKViaDocker() { + DOCKER_PPODMAN="${1}" + # TODO This could be extracted overridden by the user if we support more # architectures going forwards local container_architecture="x86_64/ubuntu" diff --git a/makejdk-any-platform.1 b/makejdk-any-platform.1 index c095de133..2f51df885 100755 --- a/makejdk-any-platform.1 +++ b/makejdk-any-platform.1 @@ -22,7 +22,7 @@ that you are building for further details. \fBExample:\fR The simplest use case to run is: -"./makejdk-any-platform.sh --docker jdk8u" +"./makejdk-any-platform.sh -D jdk8u" This will start a Docker container and build you the latest Java 8 Temurin binary from the source at https://github.com/adoptium/openjdk-jdk8u @@ -97,8 +97,8 @@ specify the location for the built binary, e.g. /path/. This is typically used in conjunction with \fB<-T>\fR to create a custom path / file name for the resulting binary. .TP -.BR \-D ", " \-\-docker -build OpenJDK in a docker container. +.BR \-D ", " \-\-docker ", " \-\-podman +build OpenJDK in a docker/podman container. -D will autodetect, using podman if found, docker otherwise. .TP .BR \-\-debug-docker debug OpenJDK build script in a docker container. Only valid if \fB-D\fR is selected. diff --git a/makejdk-any-platform.sh b/makejdk-any-platform.sh index 2dc01423f..b37f7b641 100755 --- a/makejdk-any-platform.sh +++ b/makejdk-any-platform.sh @@ -64,8 +64,8 @@ done echo "${makeJdkArgs}" > ./workspace/config/makejdk-any-platform.args # Let's build and test the (Adoptium) OpenJDK binary in Docker or natively -if [ "${BUILD_CONFIG[USE_DOCKER]}" == "true" ] ; then - buildOpenJDKViaDocker +if [ ! "${BUILD_CONFIG[USE_DOCKER]}" == "false" ] ; then + buildOpenJDKViaDocker ${BUILD_CONFIG[USE_DOCKER]} else buildOpenJDKInNativeEnvironment fi diff --git a/sbin/build.sh b/sbin/build.sh index 5f7fbfad2..21c5f49dd 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -2100,7 +2100,7 @@ createTargetDir() { fixJavaHomeUnderDocker() { # If we are inside docker we cannot trust the JDK_BOOT_DIR that was detected on the host system - if [[ "${BUILD_CONFIG[USE_DOCKER]}" == "true" ]]; then + if [[ ! "${BUILD_CONFIG[USE_DOCKER]}" == "false" ]]; then # clear BUILD_CONFIG[JDK_BOOT_DIR] and re set it BUILD_CONFIG[JDK_BOOT_DIR]="" setBootJdk diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index cd3e82878..309ba1a8d 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -273,8 +273,14 @@ function parseConfigurationArguments() { "--destination" | "-d" ) BUILD_CONFIG[TARGET_DIR]="$1"; shift;; - "--docker" | "-D" ) - BUILD_CONFIG[USE_DOCKER]="true";; + "-D" ) + if [ which podman ] ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi;; + + "--docker" ) + BUILD_CONFIG[USE_DOCKER]="docker";; + + "--podman" ) + BUILD_CONFIG[USE_DOCKER]="podman";; "--debug-docker" ) BUILD_CONFIG[DEBUG_DOCKER]="true";; From 04b28dc4630cabb223c94c01b23186798cabceb6 Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 26 Apr 2024 19:10:39 +0200 Subject: [PATCH 02/30] Adjusted signalHandler to countwith podman/docker Not sure if it is used: KEEP_CONTAINER and $CONTAINER_NAME are nowhere to found however BUILD_CONFIG[KEEP_CONTAINER] and BUILD_CONFIG[CONTAINER_NAME] are thus using them and BUILD_CONFIG[USE_DOCKER] for command handler --- signalhandler.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalhandler.sh b/signalhandler.sh index ba5beaff0..09df0ec33 100755 --- a/signalhandler.sh +++ b/signalhandler.sh @@ -13,8 +13,8 @@ # ******************************************************************************** exit_script() { - if [[ -z ${KEEP_CONTAINER} ]] ; then - docker ps -a | awk '{ print $1,$2 }' | grep "$CONTAINER_NAME" | awk '{print $1 }' | xargs -I {} docker rm -f {} + if [[ -z "${BUILD_CONFIG[KEEP_CONTAINER]}" ]] ; then + "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[USE_DOCKER]}" rm -f {} fi echo "Process exited" trap - SIGINT SIGTERM # clear the trap From 6b43591f8a16d19c7eb020b24eea22571e4942c0 Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 26 Apr 2024 19:44:51 +0200 Subject: [PATCH 03/30] reworked BUILD_CONFIG[DOCKER] to contain only sudo information it is not used consitently anyway, there is BUILD_CONFIG[DOCKER] x plain docker. It will be utilised to ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} where commands are same. Where not (eg buildah), ${BUILD_CONFIG[DOCKER]} will be used as needed Once it is unified, it would be worth to rename BUILD_CONFIG[DOCKER] to BUILD_CONFIG[CONTAINER_WITH_SUDO] BUILD_CONFIG[USE_DOCKER] to BUILD_CONFIG[CONTAINER_PROVIDER] --- sbin/common/config_init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 309ba1a8d..fd44b0362 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -354,7 +354,7 @@ function parseConfigurationArguments() { BUILD_CONFIG[SIGN]=true; BUILD_CONFIG[CERTIFICATE]="$1"; shift;; "--sudo" ) - BUILD_CONFIG[DOCKER]="sudo docker";; + BUILD_CONFIG[DOCKER]="sudo";; "--tag" | "-t" ) BUILD_CONFIG[TAG]="$1"; BUILD_CONFIG[SHALLOW_CLONE_OPTION]=""; shift;; @@ -606,7 +606,7 @@ function configDefaults() { # Whether to use Temurin's cacerts file (true) or use the file provided by OpenJDK (false) BUILD_CONFIG[CUSTOM_CACERTS]=${BUILD_CONFIG[CUSTOM_CACERTS]:-"true"} - BUILD_CONFIG[DOCKER]=${BUILD_CONFIG[DOCKER]:-"docker"} + BUILD_CONFIG[DOCKER]=${BUILD_CONFIG[DOCKER]:-""} BUILD_CONFIG[TMP_SPACE_BUILD]=${BUILD_CONFIG[TMP_SPACE_BUILD]:-false} From ce82463460f70c26e11a1ac1da1804648bf50ace Mon Sep 17 00:00:00 2001 From: Jiri Date: Sun, 28 Apr 2024 10:27:59 +0200 Subject: [PATCH 04/30] using the BUILD_CONFIG[DOCKER] BUILD_CONFIG[USE_DOCKER] combo proeprly --- docker-build.sh | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index 62a46528d..c44a362a6 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -39,12 +39,16 @@ createPersistentDockerDataVolume() # shellcheck disable=SC2154 echo "Removing old volumes and containers" # shellcheck disable=SC2046 - ${BUILD_CONFIG[DOCKER]} rm -f $(${BUILD_CONFIG[DOCKER]} ps -a --no-trunc -q -f volume="${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}") || true - ${BUILD_CONFIG[DOCKER]} volume rm -f "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" || true + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} rm -f $(${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} ps -a --no-trunc -q -f volume="${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}") || true + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} volume rm -f "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" || true # shellcheck disable=SC2154 echo "Creating tmp container" - ${BUILD_CONFIG[DOCKER]} volume create --name "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + if echo ${BUILD_CONFIG[USE_DOCKER]} | grep docker ; then + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} volume create --name "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + else + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} volume create "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + fi fi } @@ -64,7 +68,7 @@ buildDockerContainer() writeConfigToFile - ${BUILD_CONFIG[DOCKER]} build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" } # Execute the (Adoptium) OpenJDK build inside the Docker Container @@ -185,11 +189,26 @@ buildOpenJDKViaDocker() fi # Command without gitSshAccess or dockerMode arrays + local pipelinesdir="${hostDir}"/workspace/pipelines + if [ -e "${hostDir}"/pipelines ] ; then + local pipelinesdir="${hostDir}"/pipelines + else + mkdir -p "${pipelinesdir}" + fi + if echo ${BUILD_CONFIG[USE_DOCKER]} | grep docker ; then + local cpuset="--cpuset-cpus=${cpuSet}" + else + local cpuset="" + fi + local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then jsut podman/docker in USE_DOCKER? + local targetdir="${hostDir}"/workspace/target + mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? + echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( - "--cpuset-cpus=${cpuSet}" + ${cpuset} -v "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}:/openjdk/build" - -v "${hostDir}"/workspace/target:/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":Z - -v "${hostDir}"/pipelines:/openjdk/pipelines:Z + -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":$mountflag + -v "${pipelinesdir}":/openjdk/pipelines:$mountflag -e "DEBUG_DOCKER_FLAG=${BUILD_CONFIG[DEBUG_DOCKER]}" -e "BUILD_VARIANT=${BUILD_CONFIG[BUILD_VARIANT]}" "${dockerEntrypoint[@]:+${dockerEntrypoint[@]}}") @@ -206,14 +225,14 @@ buildOpenJDKViaDocker() fi # Run the command string in Docker - ${BUILD_CONFIG[DOCKER]} run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" # Tell user where the resulting binary can be found on the host system - echo "The finished image can be found in ${hostDir}/workspace/target on the host system" + echo "The finished image can be found in ${targetdir} on the host system" # If we didn't specify to keep the container then remove it if [[ "${BUILD_CONFIG[KEEP_CONTAINER]}" == "false" ]] ; then echo "Removing container ${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" - ${BUILD_CONFIG[DOCKER]} ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[DOCKER]}" rm {} + ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[DOCKER]}" ${BUILD_CONFIG[USE_DOCKER] rm {} fi } From 9e653f303024db16f9e05b71601839ead388c4bc Mon Sep 17 00:00:00 2001 From: Jiri Date: Tue, 7 May 2024 17:18:08 +0200 Subject: [PATCH 05/30] On podman, set --userns=keep-id --- docker-build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-build.sh b/docker-build.sh index c44a362a6..6a987cf62 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -200,12 +200,18 @@ buildOpenJDKViaDocker() else local cpuset="" fi + if echo ${BUILD_CONFIG[USE_DOCKER]} | grep podman ; then + local userns="--userns=keep-id" + else + local userns="" + fi local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then jsut podman/docker in USE_DOCKER? local targetdir="${hostDir}"/workspace/target mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( ${cpuset} + ${userns} -v "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}:/openjdk/build" -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":$mountflag -v "${pipelinesdir}":/openjdk/pipelines:$mountflag From 743f470fd8ddeb2ac041a4336397f02828162bf5 Mon Sep 17 00:00:00 2001 From: Jiri Date: Tue, 7 May 2024 17:40:45 +0200 Subject: [PATCH 06/30] porecreate all necessary dirs Podman is creating all mounted folders as root root 744 So next to --userns=keep-id which set proeprly the owner ow mounted folder itself, we have to pre-create the used parents of mounted folder Maybe this should be podman only, but afaik it do not hurt in docker --- docker-build.sh | 14 ++++++++------ docker/dockerfile-generator.sh | 19 ++++++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index 6a987cf62..3e9c02a5e 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -77,6 +77,12 @@ buildOpenJDKViaDocker() DOCKER_PPODMAN="${1}" + local hostDir + hostDir="$(pwd)" + local pipelinesdir="${hostDir}"/workspace/pipelines + local workspacedir="${hostDir}"/workspace # we must ensure build user have correct permissions here + local targetdir="${hostDir}"/workspace/target + # TODO This could be extracted overridden by the user if we support more # architectures going forwards local container_architecture="x86_64/ubuntu" @@ -86,7 +92,8 @@ buildOpenJDKViaDocker() if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "openj9" ]; then build_variant_flag="--openj9" fi - docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" + docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ + --dirs "${workspacedir} ${targetdir}" # shellcheck disable=SC1090,SC1091 source "${BUILD_CONFIG[DOCKER_FILE_PATH]}/dockerConfiguration.sh" @@ -167,9 +174,6 @@ buildOpenJDKViaDocker() # Show the user all of the config before we build displayParams - local hostDir - hostDir="$(pwd)" - echo "Target binary directory on host machine: ${hostDir}/target" mkdir -p "${hostDir}/workspace/target" @@ -189,7 +193,6 @@ buildOpenJDKViaDocker() fi # Command without gitSshAccess or dockerMode arrays - local pipelinesdir="${hostDir}"/workspace/pipelines if [ -e "${hostDir}"/pipelines ] ; then local pipelinesdir="${hostDir}"/pipelines else @@ -206,7 +209,6 @@ buildOpenJDKViaDocker() local userns="" fi local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then jsut podman/docker in USE_DOCKER? - local targetdir="${hostDir}"/workspace/target mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 4865df20a..afe4eba62 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -18,6 +18,7 @@ set -eu OPENJ9=false BUILD=false COMMENTS=false +DIRS= PRINT=false DOCKERFILE_DIR= DOCKERFILE_PATH= @@ -87,6 +88,11 @@ processArgs() { COMMENTS=true shift ;; + --dirs) + DIRS="${2}" + shift + shift + ;; --path) DOCKERFILE_DIR=$2 shift @@ -135,6 +141,7 @@ usage() { --build Build the docker image after generation and create interactive container --clean Remove all dockerfiles (Dockerfile*) from '--path' --comments Prints comments into the dockerfile + --dirs space separated list of dirs to be created, with proper permissions --path Specify where to save the dockerfile (Default: $PWD) --print Print the Dockerfile to screen after generation --openj9 Make the Dockerfile able to build w/OpenJ9 JIT @@ -264,6 +271,14 @@ printgcc() { ENV CC=gcc-7 CXX=g++-7" >> "$DOCKERFILE_PATH" } +printCustomDirs() { + for dir in ${DIRS} ; do + echo "RUN mkdir -p $dir" >> "$DOCKERFILE_PATH" + echo "RUN chmod 755 $dir" >> "$DOCKERFILE_PATH" + echo "RUN chown -R build $dir" >> "$DOCKERFILE_PATH" + done +} + printDockerJDKs() { # JDK8 uses zulu-7 to as it's bootjdk if [ "${JDK_VERSION}" != 8 ] && [ "${JDK_VERSION}" != "${JDK_MAX}" ]; then @@ -339,7 +354,9 @@ ARG HostUID ENV HostUID=\$HostUID RUN useradd -u \$HostUID -ms /bin/bash build WORKDIR /openjdk/build -RUN chown -R build /openjdk/ +RUN chown -R build /openjdk/" >> "$DOCKERFILE_PATH" + printCustomDirs + echo " USER build" >> "$DOCKERFILE_PATH" } From 480aa29216e50b4404e2961c6b6058f009577978 Mon Sep 17 00:00:00 2001 From: Jiri Date: Tue, 7 May 2024 17:47:45 +0200 Subject: [PATCH 07/30] Added warinbg to `docker build` command when used with podman --- docker/dockerfile-generator.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index afe4eba62..5dfd65896 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -433,6 +433,7 @@ if [ "${BUILD}" == true ]; then commandString="${commandString} --build-variant openj9" fi + # although this works for both docekr and podman with docker alias, it shodl honour the setup pf BUILD_CONFIG[USE_DOCKER] (also maybe with BUILD_CONFIG[DOCKER] which set sudo/no sudo) docker build -t "jdk${JDK_VERSION}_build_image" -f "$DOCKERFILE_PATH" . --build-arg "OPENJDK_CORE_VERSION=${JDK_VERSION}" --build-arg "HostUID=${UID}" echo "To start a build run ${commandString}" docker run -it "jdk${JDK_VERSION}_build_image" bash From e3af8d27150bbb351fb3dbb83abaf2fb4606252b Mon Sep 17 00:00:00 2001 From: Jiri Date: Tue, 7 May 2024 18:55:18 +0200 Subject: [PATCH 08/30] Using absolute path instead of "." however it was not guilty: open my $fh, '<', $filename or die "Couldn't open file: $!"; in mk-ca-bundle.pl is. --- security/mk-cacerts.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/security/mk-cacerts.sh b/security/mk-cacerts.sh index 102b1934a..573a73ba7 100755 --- a/security/mk-cacerts.sh +++ b/security/mk-cacerts.sh @@ -12,6 +12,19 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** +## resolve folder of this script, following all symlinks, +## http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in +SCRIPT_SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SCRIPT_SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" && pwd )" + SCRIPT_SOURCE="$(readlink "$SCRIPT_SOURCE")" + # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $SCRIPT_SOURCE != /* ]] && SCRIPT_SOURCE="$SCRIPT_DIR/$SCRIPT_SOURCE" +done +readonly SCRIPT_DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" && pwd )" + + + set -euo pipefail PROGRAM_NAME="${0##*/}" @@ -54,7 +67,7 @@ fi # Convert Mozilla's list of certificates into a PEM file. The -n switch makes # it use the local certdata.txt in this folder. -certNum=$(./mk-ca-bundle.pl -v -n ca-bundle.crt) +certNum=$(${SCRIPT_DIR}/mk-ca-bundle.pl -v -n ca-bundle.crt) echo "mk-ca-bundle.pl generates $certNum certificates" # Split the PEM file into individual files because keytool cannot do it on its own. From 092d1920706d88ca430654d477cc36527bee5c81 Mon Sep 17 00:00:00 2001 From: judovana Date: Wed, 8 May 2024 14:36:37 +0200 Subject: [PATCH 09/30] docker-build.sh jsut-> just Co-authored-by: Stewart X Addison <6487691+sxa@users.noreply.github.com> --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 3e9c02a5e..0ff7b12eb 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -208,7 +208,7 @@ buildOpenJDKViaDocker() else local userns="" fi - local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then jsut podman/docker in USE_DOCKER? + local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then just podman/docker in USE_DOCKER? mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( From fa34c0d2869d07a1a9e408d30d4487d035b3691c Mon Sep 17 00:00:00 2001 From: Jiri Date: Wed, 8 May 2024 14:49:21 +0200 Subject: [PATCH 10/30] Removed unused parameter of buildOpenJDKViaDocker buildOpenJDKViaDocker do not need ${BUILD_CONFIG[USE_DOCKER]} as parameter. docker-build.sh is inheriting the whole BUILD_CONFIG --- docker-build.sh | 3 --- makejdk-any-platform.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index 0ff7b12eb..aa0a78264 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -74,9 +74,6 @@ buildDockerContainer() # Execute the (Adoptium) OpenJDK build inside the Docker Container buildOpenJDKViaDocker() { - - DOCKER_PPODMAN="${1}" - local hostDir hostDir="$(pwd)" local pipelinesdir="${hostDir}"/workspace/pipelines diff --git a/makejdk-any-platform.sh b/makejdk-any-platform.sh index b37f7b641..9531855c1 100755 --- a/makejdk-any-platform.sh +++ b/makejdk-any-platform.sh @@ -65,7 +65,7 @@ echo "${makeJdkArgs}" > ./workspace/config/makejdk-any-platform.args # Let's build and test the (Adoptium) OpenJDK binary in Docker or natively if [ ! "${BUILD_CONFIG[USE_DOCKER]}" == "false" ] ; then - buildOpenJDKViaDocker ${BUILD_CONFIG[USE_DOCKER]} + buildOpenJDKViaDocker else buildOpenJDKInNativeEnvironment fi From 3d2a7a58b3ff4639570a115ba6a55c02ab0b450d Mon Sep 17 00:00:00 2001 From: judovana Date: Thu, 9 May 2024 15:27:12 +0200 Subject: [PATCH 11/30] use which podman without [] and to dev/null Co-authored-by: Stewart X Addison <6487691+sxa@users.noreply.github.com> --- sbin/common/config_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index fd44b0362..0467cbd95 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -274,7 +274,7 @@ function parseConfigurationArguments() { BUILD_CONFIG[TARGET_DIR]="$1"; shift;; "-D" ) - if [ which podman ] ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi;; + if which podman > /dev/null ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi;; "--docker" ) BUILD_CONFIG[USE_DOCKER]="docker";; From 20f6965f131e26c47ced9b82f091ceceec54921b Mon Sep 17 00:00:00 2001 From: Jiri Date: Thu, 9 May 2024 15:28:31 +0200 Subject: [PATCH 12/30] Revert "Using absolute path instead of "."" This reverts commit 55f1195eec2b90a16bcab15795fd6f1bbb8d9921. --- security/mk-cacerts.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/security/mk-cacerts.sh b/security/mk-cacerts.sh index 573a73ba7..102b1934a 100755 --- a/security/mk-cacerts.sh +++ b/security/mk-cacerts.sh @@ -12,19 +12,6 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** -## resolve folder of this script, following all symlinks, -## http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in -SCRIPT_SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SCRIPT_SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - SCRIPT_DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" && pwd )" - SCRIPT_SOURCE="$(readlink "$SCRIPT_SOURCE")" - # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - [[ $SCRIPT_SOURCE != /* ]] && SCRIPT_SOURCE="$SCRIPT_DIR/$SCRIPT_SOURCE" -done -readonly SCRIPT_DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" && pwd )" - - - set -euo pipefail PROGRAM_NAME="${0##*/}" @@ -67,7 +54,7 @@ fi # Convert Mozilla's list of certificates into a PEM file. The -n switch makes # it use the local certdata.txt in this folder. -certNum=$(${SCRIPT_DIR}/mk-ca-bundle.pl -v -n ca-bundle.crt) +certNum=$(./mk-ca-bundle.pl -v -n ca-bundle.crt) echo "mk-ca-bundle.pl generates $certNum certificates" # Split the PEM file into individual files because keytool cannot do it on its own. From 4db5724bc3aa863a18b0beff35f9501c1ddbb483 Mon Sep 17 00:00:00 2001 From: Jiri Date: Thu, 9 May 2024 18:29:56 +0200 Subject: [PATCH 13/30] Fixed issue with missing test for mk-ca-bundle.pl --- docker/dockerfile-generator.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 5dfd65896..f3b982750 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -340,6 +340,7 @@ printCopyFolders(){ echo " COPY sbin /openjdk/sbin COPY security /openjdk/security +COPY test /openjdk/test COPY workspace/config /openjdk/config" >> "$DOCKERFILE_PATH" } From 04f44abcf5f84b5467ee179b06fb906d1285ddfb Mon Sep 17 00:00:00 2001 From: Jiri Date: Thu, 9 May 2024 19:15:41 +0200 Subject: [PATCH 14/30] Always generate configure-and-build.sh --- sbin/common/config_init.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 0467cbd95..0fe9392cf 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -274,13 +274,16 @@ function parseConfigurationArguments() { BUILD_CONFIG[TARGET_DIR]="$1"; shift;; "-D" ) - if which podman > /dev/null ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi;; + if which podman > /dev/null ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi; + BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;; "--docker" ) - BUILD_CONFIG[USE_DOCKER]="docker";; + BUILD_CONFIG[USE_DOCKER]="docker"; + BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;; "--podman" ) - BUILD_CONFIG[USE_DOCKER]="podman";; + BUILD_CONFIG[USE_DOCKER]="podman"; + BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;; "--debug-docker" ) BUILD_CONFIG[DEBUG_DOCKER]="true";; From 56df9083ab2baa21d74add66f0166729a704f5e5 Mon Sep 17 00:00:00 2001 From: Jiri Vanek Date: Mon, 13 May 2024 15:23:51 +0200 Subject: [PATCH 15/30] Removed more hardcoded dockers --- docker-build.sh | 4 ++-- docker/dockerfile-generator.sh | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index aa0a78264..5c2d6ccdf 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -90,7 +90,7 @@ buildOpenJDKViaDocker() build_variant_flag="--openj9" fi docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ - --dirs "${workspacedir} ${targetdir}" + --dirs "${workspacedir} ${targetdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" # shellcheck disable=SC1090,SC1091 source "${BUILD_CONFIG[DOCKER_FILE_PATH]}/dockerConfiguration.sh" @@ -139,7 +139,7 @@ buildOpenJDKViaDocker() BUILD_CONFIG[DEBUG_IMAGE_PATH]=$openjdk_debug_image_path BUILD_CONFIG[STATIC_LIBS_IMAGE_PATH]=$static_libs_dir - if [ -z "$(command -v docker)" ]; then + if [ -z "$(command -v ${BUILD_CONFIG[USE_DOCKER]})" ]; then # shellcheck disable=SC2154 echo "Error, please install docker and ensure that it is in your path and running!" exit diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index f3b982750..59a44adbf 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -113,6 +113,11 @@ processArgs() { shift shift ;; + --command) + CMD="${2}" + shift + shift + ;; *) echo "Unrecognised Argument: $1" exit 1 @@ -132,6 +137,14 @@ processArgs() { if [ ${OPENJ9} == true ]; then DOCKERFILE_PATH="$DOCKERFILE_PATH-openj9" fi + + if [ -z "$CMD" ]; then + if which podman > /dev/null; then + CMD=podman + else + CMD=docker + fi + fi } usage() { @@ -435,7 +448,7 @@ if [ "${BUILD}" == true ]; then fi # although this works for both docekr and podman with docker alias, it shodl honour the setup pf BUILD_CONFIG[USE_DOCKER] (also maybe with BUILD_CONFIG[DOCKER] which set sudo/no sudo) - docker build -t "jdk${JDK_VERSION}_build_image" -f "$DOCKERFILE_PATH" . --build-arg "OPENJDK_CORE_VERSION=${JDK_VERSION}" --build-arg "HostUID=${UID}" + ${CMD} build -t "jdk${JDK_VERSION}_build_image" -f "$DOCKERFILE_PATH" . --build-arg "OPENJDK_CORE_VERSION=${JDK_VERSION}" --build-arg "HostUID=${UID}" echo "To start a build run ${commandString}" - docker run -it "jdk${JDK_VERSION}_build_image" bash + ${CMD} run -it "jdk${JDK_VERSION}_build_image" bash fi From a1a3d57fb67c266a62093d2b7790d3363aa0e7a9 Mon Sep 17 00:00:00 2001 From: Jiri Date: Tue, 14 May 2024 08:13:44 +0200 Subject: [PATCH 16/30] Do not set boot jdk for docekr builds (it is removed later anyway) --- configureBuild.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configureBuild.sh b/configureBuild.sh index 924863833..a16dde234 100755 --- a/configureBuild.sh +++ b/configureBuild.sh @@ -410,5 +410,7 @@ configure_build() { setWorkingDirectory configureMacFreeFont setMakeArgs - setBootJdk + if [ "${BUILD_CONFIG[USE_DOCKER]}" == false ] ; then + setBootJdk + fi } From e566202767bb888033729234057c62e8dee63835 Mon Sep 17 00:00:00 2001 From: Jiri Date: Wed, 15 May 2024 16:25:29 +0200 Subject: [PATCH 17/30] Added few more missing dirs --- docker-build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 5c2d6ccdf..8666218bf 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -79,6 +79,8 @@ buildOpenJDKViaDocker() local pipelinesdir="${hostDir}"/workspace/pipelines local workspacedir="${hostDir}"/workspace # we must ensure build user have correct permissions here local targetdir="${hostDir}"/workspace/target + local targetsrcdir="${hostDir}"/workspace/build/src + local configdir="${hostDir}"/workspace/config # TODO This could be extracted overridden by the user if we support more # architectures going forwards @@ -90,7 +92,7 @@ buildOpenJDKViaDocker() build_variant_flag="--openj9" fi docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ - --dirs "${workspacedir} ${targetdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" + --dirs "${workspacedir} ${targetdir} ${targetsrcdir} ${configdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" # shellcheck disable=SC1090,SC1091 source "${BUILD_CONFIG[DOCKER_FILE_PATH]}/dockerConfiguration.sh" @@ -214,6 +216,7 @@ buildOpenJDKViaDocker() -v "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}:/openjdk/build" -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":$mountflag -v "${pipelinesdir}":/openjdk/pipelines:$mountflag + -v "${configdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"config":$mountflag -e "DEBUG_DOCKER_FLAG=${BUILD_CONFIG[DEBUG_DOCKER]}" -e "BUILD_VARIANT=${BUILD_CONFIG[BUILD_VARIANT]}" "${dockerEntrypoint[@]:+${dockerEntrypoint[@]}}") From 18ef020fda11e9040e5c82733c7bd2f469b63239 Mon Sep 17 00:00:00 2001 From: Jiri Date: Wed, 15 May 2024 16:59:52 +0200 Subject: [PATCH 18/30] Fixed boot jdk check against new docker/podman/false --- sbin/common/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/common/common.sh b/sbin/common/common.sh index 6f544a848..8b2ea759c 100755 --- a/sbin/common/common.sh +++ b/sbin/common/common.sh @@ -233,7 +233,7 @@ createOpenJDKArchive() function setBootJdk() { # Stops setting the bootJDK on the host machine when running docker-build - if [ "${BUILD_CONFIG[DOCKER]}" != "docker" ] || { [ "${BUILD_CONFIG[DOCKER]}" == "docker" ] && [ "${BUILD_CONFIG[DOCKER_FILE_PATH]}" != "" ]; } ; then + if [ "${BUILD_CONFIG[DOCKER]}" == "false" ] || { [ "${BUILD_CONFIG[DOCKER]}" != "false" ] && [ "${BUILD_CONFIG[DOCKER_FILE_PATH]}" != "" ]; } ; then if [ -z "${BUILD_CONFIG[JDK_BOOT_DIR]}" ] ; then echo "Searching for JDK_BOOT_DIR" From f1d861181cd95f10ed422e073ad3ee6511dcf9cb Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 20 May 2024 17:21:10 +0200 Subject: [PATCH 19/30] replacing missed ${BUILD_CONFIG[DOCKER]} by ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" Originally, this patch started to fix properly quote for safety (thanx linter), I foudn that on sme pleaces, original ${BUILD_CONFIG[DOCKER]} was not repalced by new tandem. ${BUILD_CONFIG[DOCKER]} was 'docker' or 'sudo docker'. I had split it, so ${BUILD_CONFIG[DOCKER]} is sudo or nothing and ${BUILD_CONFIG[USE_DOCKER]}" is docker or podman. The variables have to be renamed at the end to adhere more to theirs purposes. --- docker-build.sh | 43 +++++++++++++++++++--------------- docker/dockerfile-generator.sh | 2 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index 8666218bf..3b87c8b00 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -20,6 +20,12 @@ # ################################################################################ +# the ${BUILD_CONFIG[DOCKER]} can not be quoted. It is sudo (or simialrly) or nothing. "" is not an option. +# simialrly the ${cpuset} and ${userns} +# shellcheck disable=SC2206 +# shellcheck disable=SC2046 +# shellcheck disable=SC2086 + set -eu # Create a data volume called ${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}, @@ -30,7 +36,7 @@ set -eu createPersistentDockerDataVolume() { set +e - ${BUILD_CONFIG[DOCKER]} volume inspect "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" > /dev/null 2>&1 + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume inspect "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" > /dev/null 2>&1 local data_volume_exists=$? set -e @@ -38,16 +44,15 @@ createPersistentDockerDataVolume() # shellcheck disable=SC2154 echo "Removing old volumes and containers" - # shellcheck disable=SC2046 - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} rm -f $(${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} ps -a --no-trunc -q -f volume="${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}") || true - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} volume rm -f "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" || true + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" rm -f $(${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a --no-trunc -q -f volume="${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}") || true + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume rm -f "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" || true # shellcheck disable=SC2154 echo "Creating tmp container" - if echo ${BUILD_CONFIG[USE_DOCKER]} | grep docker ; then - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} volume create --name "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + if echo "${BUILD_CONFIG[USE_DOCKER]}" | grep docker ; then + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume create --name "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" else - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} volume create "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume create "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" fi fi } @@ -68,7 +73,7 @@ buildDockerContainer() writeConfigToFile - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" } # Execute the (Adoptium) OpenJDK build inside the Docker Container @@ -141,7 +146,7 @@ buildOpenJDKViaDocker() BUILD_CONFIG[DEBUG_IMAGE_PATH]=$openjdk_debug_image_path BUILD_CONFIG[STATIC_LIBS_IMAGE_PATH]=$static_libs_dir - if [ -z "$(command -v ${BUILD_CONFIG[USE_DOCKER]})" ]; then + if [ -z "$(command -v "${BUILD_CONFIG[USE_DOCKER]}")" ]; then # shellcheck disable=SC2154 echo "Error, please install docker and ensure that it is in your path and running!" exit @@ -156,7 +161,7 @@ buildOpenJDKViaDocker() if [[ "${BUILD_CONFIG[REUSE_CONTAINER]}" == "true" ]] ; then # shellcheck disable=SC2086 # If we can't find the previous Docker container then build a new one - if [ "$(${BUILD_CONFIG[DOCKER]} ps -a | grep -c \"${BUILD_CONFIG[CONTAINER_NAME]}\")" == 0 ]; then + if [ "$(${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} ps -a | grep -c \"${BUILD_CONFIG[CONTAINER_NAME]}\")" == 0 ]; then echo "No docker container for reuse was found, so creating '${BUILD_CONFIG[CONTAINER_NAME]}' " buildDockerContainer fi @@ -164,7 +169,7 @@ buildOpenJDKViaDocker() # shellcheck disable=SC2154 echo "Since you specified --ignore-container, we are removing the existing container (if it exists) and building you a new one{$good}" # Find the previous Docker container and remove it (if it exists) - ${BUILD_CONFIG[DOCKER]} ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[DOCKER]}" rm -f {} + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" rm -f {} # Build a new container buildDockerContainer @@ -197,26 +202,26 @@ buildOpenJDKViaDocker() else mkdir -p "${pipelinesdir}" fi - if echo ${BUILD_CONFIG[USE_DOCKER]} | grep docker ; then + if echo "${BUILD_CONFIG[USE_DOCKER]}" | grep docker ; then local cpuset="--cpuset-cpus=${cpuSet}" else local cpuset="" fi - if echo ${BUILD_CONFIG[USE_DOCKER]} | grep podman ; then + if echo "${BUILD_CONFIG[USE_DOCKER]}" | grep podman ; then local userns="--userns=keep-id" else local userns="" fi local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then just podman/docker in USE_DOCKER? mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? - echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" + echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( ${cpuset} ${userns} -v "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}:/openjdk/build" - -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":$mountflag - -v "${pipelinesdir}":/openjdk/pipelines:$mountflag - -v "${configdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"config":$mountflag + -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":"${mountflag}" + -v "${pipelinesdir}":/openjdk/pipelines:"${mountflag}" + -v "${configdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"config":"${mountflag}" -e "DEBUG_DOCKER_FLAG=${BUILD_CONFIG[DEBUG_DOCKER]}" -e "BUILD_VARIANT=${BUILD_CONFIG[BUILD_VARIANT]}" "${dockerEntrypoint[@]:+${dockerEntrypoint[@]}}") @@ -233,7 +238,7 @@ buildOpenJDKViaDocker() fi # Run the command string in Docker - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" # Tell user where the resulting binary can be found on the host system echo "The finished image can be found in ${targetdir} on the host system" @@ -241,6 +246,6 @@ buildOpenJDKViaDocker() # If we didn't specify to keep the container then remove it if [[ "${BUILD_CONFIG[KEEP_CONTAINER]}" == "false" ]] ; then echo "Removing container ${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" - ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[DOCKER]}" ${BUILD_CONFIG[USE_DOCKER] rm {} + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER] rm {} fi } diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 59a44adbf..8901d0e56 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -148,7 +148,7 @@ processArgs() { } usage() { - echo" Usage: ./dockerfile_generator.sh [OPTIONS] + echo " Usage: ./dockerfile_generator.sh [OPTIONS] Options: --help | -h Print this message and exit --build Build the docker image after generation and create interactive container From ae65c64a6adbd5ca90fe2667670d7f39b1f5916c Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 24 May 2024 13:49:11 +0200 Subject: [PATCH 20/30] instead of workspace/build/src creatig directly /workspace/build all sub dirs should be then created by follwoing prepare-workspace --- docker-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index 3b87c8b00..06f045ffe 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -84,7 +84,7 @@ buildOpenJDKViaDocker() local pipelinesdir="${hostDir}"/workspace/pipelines local workspacedir="${hostDir}"/workspace # we must ensure build user have correct permissions here local targetdir="${hostDir}"/workspace/target - local targetsrcdir="${hostDir}"/workspace/build/src + local targetbuilddir="${hostDir}"/workspace/build local configdir="${hostDir}"/workspace/config # TODO This could be extracted overridden by the user if we support more @@ -97,7 +97,7 @@ buildOpenJDKViaDocker() build_variant_flag="--openj9" fi docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ - --dirs "${workspacedir} ${targetdir} ${targetsrcdir} ${configdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" + --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" # shellcheck disable=SC1090,SC1091 source "${BUILD_CONFIG[DOCKER_FILE_PATH]}/dockerConfiguration.sh" From 67f3187a38e417e4a6264fd838dae1a29b287a4d Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 24 May 2024 13:54:13 +0200 Subject: [PATCH 21/30] Remoed accident tab --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 06f045ffe..5b17d914f 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -214,7 +214,7 @@ buildOpenJDKViaDocker() fi local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then just podman/docker in USE_DOCKER? mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? - echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" + echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( ${cpuset} ${userns} From bab4397fe0b5033ee4028164bde378127beca02c Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 24 May 2024 15:00:00 +0200 Subject: [PATCH 22/30] Added support for building local dir/src tarball in contianer --- docker-build.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 5b17d914f..2d2125039 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -86,6 +86,12 @@ buildOpenJDKViaDocker() local targetdir="${hostDir}"/workspace/target local targetbuilddir="${hostDir}"/workspace/build local configdir="${hostDir}"/workspace/config + local localsourcesdir= + + if [ "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE]}" = "true" ] ; then + # OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH can be file, you can nto mount file + localsourcesdir=$(dirname "${BUILD_CONFIG[OPENJDK_LOCAL_SOURCE_ARCHIVE_ABSPATH]}") + fi # TODO This could be extracted overridden by the user if we support more # architectures going forwards @@ -97,7 +103,7 @@ buildOpenJDKViaDocker() build_variant_flag="--openj9" fi docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ - --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" + --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir} ${localsourcesdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" # shellcheck disable=SC1090,SC1091 source "${BUILD_CONFIG[DOCKER_FILE_PATH]}/dockerConfiguration.sh" @@ -214,10 +220,15 @@ buildOpenJDKViaDocker() fi local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then just podman/docker in USE_DOCKER? mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? + local localsourcesdirmount= + if [ ! -z "${localsourcesdir}" ] ; then + localsourcesdirmount="-v ${localsourcesdir}:${localsourcesdir}:${mountflag}" #read only? Is copied anwya + fi echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" local commandString=( ${cpuset} ${userns} + ${localsourcesdirmount} -v "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}:/openjdk/build" -v "${targetdir}":/"${BUILD_CONFIG[WORKSPACE_DIR]}"/"${BUILD_CONFIG[TARGET_DIR]}":"${mountflag}" -v "${pipelinesdir}":/openjdk/pipelines:"${mountflag}" From 73c43c513e6e50916fb92f81e52a2eb7e92b2604 Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 24 May 2024 15:11:28 +0200 Subject: [PATCH 23/30] Removed wrongly added ASSEMBLE_EXPLODED_IMAGE=true to container builds --- sbin/common/config_init.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 0fe9392cf..0467cbd95 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -274,16 +274,13 @@ function parseConfigurationArguments() { BUILD_CONFIG[TARGET_DIR]="$1"; shift;; "-D" ) - if which podman > /dev/null ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi; - BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;; + if which podman > /dev/null ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi;; "--docker" ) - BUILD_CONFIG[USE_DOCKER]="docker"; - BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;; + BUILD_CONFIG[USE_DOCKER]="docker";; "--podman" ) - BUILD_CONFIG[USE_DOCKER]="podman"; - BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;; + BUILD_CONFIG[USE_DOCKER]="podman";; "--debug-docker" ) BUILD_CONFIG[DEBUG_DOCKER]="true";; From 160b76212ef8acf4d2e29f9f2eee912394f8c317 Mon Sep 17 00:00:00 2001 From: Jiri Date: Fri, 24 May 2024 15:57:22 +0200 Subject: [PATCH 24/30] used -n instead of '! -z ' --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 2d2125039..f1c8acade 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -221,7 +221,7 @@ buildOpenJDKViaDocker() local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then just podman/docker in USE_DOCKER? mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? local localsourcesdirmount= - if [ ! -z "${localsourcesdir}" ] ; then + if [ -n "${localsourcesdir}" ] ; then localsourcesdirmount="-v ${localsourcesdir}:${localsourcesdir}:${mountflag}" #read only? Is copied anwya fi echo "If you get permissions denied on ${targetdir} or ${pipelinesdir} try to turn off selinux" From 51ccc932003ba071efdac85d6543e73438c7be5e Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 17 Jun 2024 17:24:19 +0200 Subject: [PATCH 25/30] Highlighted sudo for dcoekr --- makejdk-any-platform.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makejdk-any-platform.1 b/makejdk-any-platform.1 index 2f51df885..4a1587a2a 100755 --- a/makejdk-any-platform.1 +++ b/makejdk-any-platform.1 @@ -220,6 +220,6 @@ specify the JVM variant (server or client), defaults to server. Some common example usages: -"./makejdk-any-platform --docker jdk8u" +"./makejdk-any-platform --sudo --docker jdk8u" "./makejdk-any-platform -s /home/openjdk10/src -d /home/openjdk/target -T MyOpenJDK10.tar.gz jdk10" From 35201e2094e9a38260bb385c9589f1c7dbbee504 Mon Sep 17 00:00:00 2001 From: Jiri Vanek Date: Tue, 18 Jun 2024 12:02:22 +0200 Subject: [PATCH 26/30] Added missing bracket --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index f1c8acade..cc773b18c 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -257,6 +257,6 @@ buildOpenJDKViaDocker() # If we didn't specify to keep the container then remove it if [[ "${BUILD_CONFIG[KEEP_CONTAINER]}" == "false" ]] ; then echo "Removing container ${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER] rm {} + ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} rm {} fi } From e3835549d91caf76214ace9a89c314d6a2371f3e Mon Sep 17 00:00:00 2001 From: Jiri Vanek Date: Wed, 19 Jun 2024 12:31:57 +0200 Subject: [PATCH 27/30] Fixed typo --- docker/dockerfile-generator.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 8901d0e56..250770067 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -447,7 +447,7 @@ if [ "${BUILD}" == true ]; then commandString="${commandString} --build-variant openj9" fi - # although this works for both docekr and podman with docker alias, it shodl honour the setup pf BUILD_CONFIG[USE_DOCKER] (also maybe with BUILD_CONFIG[DOCKER] which set sudo/no sudo) + # although this works for both docekr and podman with docker alias, it shodl honour the setup of BUILD_CONFIG[USE_DOCKER] (also maybe with BUILD_CONFIG[DOCKER] which set sudo/no sudo) ${CMD} build -t "jdk${JDK_VERSION}_build_image" -f "$DOCKERFILE_PATH" . --build-arg "OPENJDK_CORE_VERSION=${JDK_VERSION}" --build-arg "HostUID=${UID}" echo "To start a build run ${commandString}" ${CMD} run -it "jdk${JDK_VERSION}_build_image" bash From f1b9336d9e32fb6d2b48fe1f368b80d7b55b0676 Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 24 Jun 2024 12:00:59 +0200 Subject: [PATCH 28/30] Mentioned issue with --custom-cacerts on podman with https://github.com/adoptium/temurin-build/issues/3862 --- makejdk-any-platform.1 | 1 + 1 file changed, 1 insertion(+) diff --git a/makejdk-any-platform.1 b/makejdk-any-platform.1 index 4a1587a2a..118702b6b 100755 --- a/makejdk-any-platform.1 +++ b/makejdk-any-platform.1 @@ -99,6 +99,7 @@ This is typically used in conjunction with \fB<-T>\fR to create a custom path .TP .BR \-D ", " \-\-docker ", " \-\-podman build OpenJDK in a docker/podman container. -D will autodetect, using podman if found, docker otherwise. +When podman is used, there is bug requiring \fI--custom-cacerts\fR to be used. It is known issue: https://github.com/adoptium/temurin-build/issues/3862 .TP .BR \-\-debug-docker debug OpenJDK build script in a docker container. Only valid if \fB-D\fR is selected. From 43af711558dc53c92bce2d8f7ef3dd7df4910bbc Mon Sep 17 00:00:00 2001 From: Jiri Date: Mon, 24 Jun 2024 13:38:07 +0200 Subject: [PATCH 29/30] renamed USE_DOCKER->CONTAINER_COMMAND DOCKER->CONTAINER_AS_ROOT BUILD_CONFIG[USE_DOCKER]-> BUILD_CONFIG[CONTAINER_COMMAND] BUILD_CONFIG[DOCKER] -> BUILD_CONFIG[CONTAINER_AS_ROOT] BUILD_CONFIG[USE_DOCKER] values: false, podman, docker BUILD_CONFIG[DOCKER] values: sudo,empty string Other docker based variables which are globally container bound remained intact (CLEAN_DOCKER_BUILD, DEBUG_DOCKER, DOCKER_FILE_PATH...) --- configureBuild.sh | 4 ++-- cyclonedx-lib/build.xml | 2 +- docker-build.sh | 34 +++++++++++++++++----------------- docker/dockerfile-generator.sh | 2 +- makejdk-any-platform.sh | 2 +- sbin/build.sh | 4 ++-- sbin/common/common.sh | 2 +- sbin/common/config_init.sh | 16 ++++++++-------- signalhandler.sh | 2 +- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/configureBuild.sh b/configureBuild.sh index a16dde234..8606be3ec 100755 --- a/configureBuild.sh +++ b/configureBuild.sh @@ -86,7 +86,7 @@ doAnyBuildVariantOverrides() { # Set the working directory for this build setWorkingDirectory() { if [ -z "${BUILD_CONFIG[WORKSPACE_DIR]}" ]; then - if [[ "${BUILD_CONFIG[USE_DOCKER]}" == "true" ]]; then + if [[ "${BUILD_CONFIG[CONTAINER_COMMAND]}" == "true" ]]; then BUILD_CONFIG[WORKSPACE_DIR]="/openjdk/" else BUILD_CONFIG[WORKSPACE_DIR]="$PWD/workspace" @@ -410,7 +410,7 @@ configure_build() { setWorkingDirectory configureMacFreeFont setMakeArgs - if [ "${BUILD_CONFIG[USE_DOCKER]}" == false ] ; then + if [ "${BUILD_CONFIG[CONTAINER_COMMAND]}" == false ] ; then setBootJdk fi } diff --git a/cyclonedx-lib/build.xml b/cyclonedx-lib/build.xml index eeba51bb1..6e3e58f07 100644 --- a/cyclonedx-lib/build.xml +++ b/cyclonedx-lib/build.xml @@ -354,7 +354,7 @@ - + diff --git a/docker-build.sh b/docker-build.sh index cc773b18c..b7211f464 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -20,7 +20,7 @@ # ################################################################################ -# the ${BUILD_CONFIG[DOCKER]} can not be quoted. It is sudo (or simialrly) or nothing. "" is not an option. +# the ${BUILD_CONFIG[CONTAINER_AS_ROOT]} can not be quoted. It is sudo (or simialrly) or nothing. "" is not an option. # simialrly the ${cpuset} and ${userns} # shellcheck disable=SC2206 # shellcheck disable=SC2046 @@ -36,7 +36,7 @@ set -eu createPersistentDockerDataVolume() { set +e - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume inspect "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" > /dev/null 2>&1 + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" volume inspect "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" > /dev/null 2>&1 local data_volume_exists=$? set -e @@ -44,15 +44,15 @@ createPersistentDockerDataVolume() # shellcheck disable=SC2154 echo "Removing old volumes and containers" - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" rm -f $(${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a --no-trunc -q -f volume="${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}") || true - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume rm -f "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" || true + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" rm -f $(${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" ps -a --no-trunc -q -f volume="${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}") || true + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" volume rm -f "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" || true # shellcheck disable=SC2154 echo "Creating tmp container" - if echo "${BUILD_CONFIG[USE_DOCKER]}" | grep docker ; then - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume create --name "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + if echo "${BUILD_CONFIG[CONTAINER_COMMAND]}" | grep docker ; then + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" volume create --name "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" else - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" volume create "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" volume create "${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]}" fi fi } @@ -73,7 +73,7 @@ buildDockerContainer() writeConfigToFile - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" build -t "${BUILD_CONFIG[CONTAINER_NAME]}" -f "${dockerFile}" . --build-arg "OPENJDK_CORE_VERSION=${BUILD_CONFIG[OPENJDK_CORE_VERSION]}" --build-arg "HostUID=${UID}" } # Execute the (Adoptium) OpenJDK build inside the Docker Container @@ -103,7 +103,7 @@ buildOpenJDKViaDocker() build_variant_flag="--openj9" fi docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ - --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir} ${localsourcesdir}" --command "${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]}" + --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir} ${localsourcesdir}" --command "${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]}" # shellcheck disable=SC1090,SC1091 source "${BUILD_CONFIG[DOCKER_FILE_PATH]}/dockerConfiguration.sh" @@ -152,7 +152,7 @@ buildOpenJDKViaDocker() BUILD_CONFIG[DEBUG_IMAGE_PATH]=$openjdk_debug_image_path BUILD_CONFIG[STATIC_LIBS_IMAGE_PATH]=$static_libs_dir - if [ -z "$(command -v "${BUILD_CONFIG[USE_DOCKER]}")" ]; then + if [ -z "$(command -v "${BUILD_CONFIG[CONTAINER_COMMAND]}")" ]; then # shellcheck disable=SC2154 echo "Error, please install docker and ensure that it is in your path and running!" exit @@ -167,7 +167,7 @@ buildOpenJDKViaDocker() if [[ "${BUILD_CONFIG[REUSE_CONTAINER]}" == "true" ]] ; then # shellcheck disable=SC2086 # If we can't find the previous Docker container then build a new one - if [ "$(${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} ps -a | grep -c \"${BUILD_CONFIG[CONTAINER_NAME]}\")" == 0 ]; then + if [ "$(${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]} ps -a | grep -c \"${BUILD_CONFIG[CONTAINER_NAME]}\")" == 0 ]; then echo "No docker container for reuse was found, so creating '${BUILD_CONFIG[CONTAINER_NAME]}' " buildDockerContainer fi @@ -175,7 +175,7 @@ buildOpenJDKViaDocker() # shellcheck disable=SC2154 echo "Since you specified --ignore-container, we are removing the existing container (if it exists) and building you a new one{$good}" # Find the previous Docker container and remove it (if it exists) - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" rm -f {} + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" rm -f {} # Build a new container buildDockerContainer @@ -208,17 +208,17 @@ buildOpenJDKViaDocker() else mkdir -p "${pipelinesdir}" fi - if echo "${BUILD_CONFIG[USE_DOCKER]}" | grep docker ; then + if echo "${BUILD_CONFIG[CONTAINER_COMMAND]}" | grep docker ; then local cpuset="--cpuset-cpus=${cpuSet}" else local cpuset="" fi - if echo "${BUILD_CONFIG[USE_DOCKER]}" | grep podman ; then + if echo "${BUILD_CONFIG[CONTAINER_COMMAND]}" | grep podman ; then local userns="--userns=keep-id" else local userns="" fi - local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[DOCKER] rather then just podman/docker in USE_DOCKER? + local mountflag=Z #rw? maybe this should be bound to root/rootles content of BUILD_CONFIG[CONTAINER_AS_ROOT] rather then just podman/docker in USE_DOCKER? mkdir -p "${hostDir}"/workspace/build # shouldnt be already there? local localsourcesdirmount= if [ -n "${localsourcesdir}" ] ; then @@ -249,7 +249,7 @@ buildOpenJDKViaDocker() fi # Run the command string in Docker - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" run --name "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" "${commandString[@]}" # Tell user where the resulting binary can be found on the host system echo "The finished image can be found in ${targetdir} on the host system" @@ -257,6 +257,6 @@ buildOpenJDKViaDocker() # If we didn't specify to keep the container then remove it if [[ "${BUILD_CONFIG[KEEP_CONTAINER]}" == "false" ]] ; then echo "Removing container ${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" - ${BUILD_CONFIG[DOCKER]} "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[DOCKER]} ${BUILD_CONFIG[USE_DOCKER]} rm {} + ${BUILD_CONFIG[CONTAINER_AS_ROOT]} "${BUILD_CONFIG[CONTAINER_COMMAND]}" ps -a | awk '{ print $1,$(NF) }' | grep "${BUILD_CONFIG[OPENJDK_CORE_VERSION]}-${BUILD_CONFIG[BUILD_VARIANT]}" | awk '{print $1 }' | xargs -I {} ${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]} rm {} fi } diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 250770067..932224e05 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -447,7 +447,7 @@ if [ "${BUILD}" == true ]; then commandString="${commandString} --build-variant openj9" fi - # although this works for both docekr and podman with docker alias, it shodl honour the setup of BUILD_CONFIG[USE_DOCKER] (also maybe with BUILD_CONFIG[DOCKER] which set sudo/no sudo) + # although this works for both docekr and podman with docker alias, it shodl honour the setup of BUILD_CONFIG[CONTAINER_COMMAND] (also maybe with BUILD_CONFIG[CONTAINER_AS_ROOT] which set sudo/no sudo) ${CMD} build -t "jdk${JDK_VERSION}_build_image" -f "$DOCKERFILE_PATH" . --build-arg "OPENJDK_CORE_VERSION=${JDK_VERSION}" --build-arg "HostUID=${UID}" echo "To start a build run ${commandString}" ${CMD} run -it "jdk${JDK_VERSION}_build_image" bash diff --git a/makejdk-any-platform.sh b/makejdk-any-platform.sh index 9531855c1..ba1527608 100755 --- a/makejdk-any-platform.sh +++ b/makejdk-any-platform.sh @@ -64,7 +64,7 @@ done echo "${makeJdkArgs}" > ./workspace/config/makejdk-any-platform.args # Let's build and test the (Adoptium) OpenJDK binary in Docker or natively -if [ ! "${BUILD_CONFIG[USE_DOCKER]}" == "false" ] ; then +if [ ! "${BUILD_CONFIG[CONTAINER_COMMAND]}" == "false" ] ; then buildOpenJDKViaDocker else buildOpenJDKInNativeEnvironment diff --git a/sbin/build.sh b/sbin/build.sh index 21c5f49dd..11b5ff835 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -955,7 +955,7 @@ generateSBoM() { # Add Build Docker image SHA1 local buildimagesha=$(cat ${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[TARGET_DIR]}/metadata/docker.txt) - # ${BUILD_CONFIG[USE_DOCKER]^} always set to false cannot rely on it. + # ${BUILD_CONFIG[CONTAINER_COMMAND]^} always set to false cannot rely on it. if [ -n "${buildimagesha}" ] && [ "${buildimagesha}" != "N.A" ]; then addSBOMMetadataProperty "${javaHome}" "${classpath}" "${sbomJson}" "Use Docker for build" "true" addSBOMMetadataTools "${javaHome}" "${classpath}" "${sbomJson}" "Docker image SHA1" "${buildimagesha}" @@ -2100,7 +2100,7 @@ createTargetDir() { fixJavaHomeUnderDocker() { # If we are inside docker we cannot trust the JDK_BOOT_DIR that was detected on the host system - if [[ ! "${BUILD_CONFIG[USE_DOCKER]}" == "false" ]]; then + if [[ ! "${BUILD_CONFIG[CONTAINER_COMMAND]}" == "false" ]]; then # clear BUILD_CONFIG[JDK_BOOT_DIR] and re set it BUILD_CONFIG[JDK_BOOT_DIR]="" setBootJdk diff --git a/sbin/common/common.sh b/sbin/common/common.sh index 8b2ea759c..6cb632802 100755 --- a/sbin/common/common.sh +++ b/sbin/common/common.sh @@ -233,7 +233,7 @@ createOpenJDKArchive() function setBootJdk() { # Stops setting the bootJDK on the host machine when running docker-build - if [ "${BUILD_CONFIG[DOCKER]}" == "false" ] || { [ "${BUILD_CONFIG[DOCKER]}" != "false" ] && [ "${BUILD_CONFIG[DOCKER_FILE_PATH]}" != "" ]; } ; then + if [ "${BUILD_CONFIG[CONTAINER_AS_ROOT]}" == "false" ] || { [ "${BUILD_CONFIG[CONTAINER_AS_ROOT]}" != "false" ] && [ "${BUILD_CONFIG[DOCKER_FILE_PATH]}" != "" ]; } ; then if [ -z "${BUILD_CONFIG[JDK_BOOT_DIR]}" ] ; then echo "Searching for JDK_BOOT_DIR" diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 0467cbd95..97ea90d12 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -41,9 +41,11 @@ BUILD_REPRODUCIBLE_DATE BUILD_TIMESTAMP BUILD_VARIANT CERTIFICATE +CONTAINER_AS_ROOT CLEAN_DOCKER_BUILD CLEAN_GIT_REPO CLEAN_LIBS +CONTAINER_COMMAND CONTAINER_NAME COPY_MACOSX_FREE_FONT_LIB_FOR_JDK_FLAG COPY_MACOSX_FREE_FONT_LIB_FOR_JRE_FLAG @@ -57,7 +59,6 @@ CROSSCOMPILE DEBUG_DOCKER DEBUG_IMAGE_PATH DISABLE_ADOPT_BRANCH_SAFETY -DOCKER DOCKER_FILE_PATH DOCKER_SOURCE_VOLUME_NAME ENABLE_SBOM_STRACE @@ -101,7 +102,6 @@ TARGET_FILE_NAME TMP_CONTAINER_NAME TMP_SPACE_BUILD USE_ADOPTIUM_DEVKIT -USE_DOCKER USE_JEP319_CERTS USE_SSH USER_SUPPLIED_CONFIGURE_ARGS @@ -274,13 +274,13 @@ function parseConfigurationArguments() { BUILD_CONFIG[TARGET_DIR]="$1"; shift;; "-D" ) - if which podman > /dev/null ; then BUILD_CONFIG[USE_DOCKER]="podman" ; else BUILD_CONFIG[USE_DOCKER]="docker" ; fi;; + if which podman > /dev/null ; then BUILD_CONFIG[CONTAINER_COMMAND]="podman" ; else BUILD_CONFIG[CONTAINER_COMMAND]="docker" ; fi;; "--docker" ) - BUILD_CONFIG[USE_DOCKER]="docker";; + BUILD_CONFIG[CONTAINER_COMMAND]="docker";; "--podman" ) - BUILD_CONFIG[USE_DOCKER]="podman";; + BUILD_CONFIG[CONTAINER_COMMAND]="podman";; "--debug-docker" ) BUILD_CONFIG[DEBUG_DOCKER]="true";; @@ -354,7 +354,7 @@ function parseConfigurationArguments() { BUILD_CONFIG[SIGN]=true; BUILD_CONFIG[CERTIFICATE]="$1"; shift;; "--sudo" ) - BUILD_CONFIG[DOCKER]="sudo";; + BUILD_CONFIG[CONTAINER_AS_ROOT]="sudo";; "--tag" | "-t" ) BUILD_CONFIG[TAG]="$1"; BUILD_CONFIG[SHALLOW_CLONE_OPTION]=""; shift;; @@ -549,7 +549,7 @@ function configDefaults() { BUILD_CONFIG[CLEAN_DOCKER_BUILD]=${BUILD_CONFIG[CLEAN_DOCKER_BUILD]:-false} # Use Docker to build (defaults to false) - BUILD_CONFIG[USE_DOCKER]=${BUILD_CONFIG[USE_DOCKER]:-false} + BUILD_CONFIG[CONTAINER_COMMAND]=${BUILD_CONFIG[CONTAINER_COMMAND]:-false} # Alow to debug docker build.sh script (dafult to false) BUILD_CONFIG[DEBUG_DOCKER]=${BUILD_CONFIG[DEBUG_DOCKER]:-false} @@ -606,7 +606,7 @@ function configDefaults() { # Whether to use Temurin's cacerts file (true) or use the file provided by OpenJDK (false) BUILD_CONFIG[CUSTOM_CACERTS]=${BUILD_CONFIG[CUSTOM_CACERTS]:-"true"} - BUILD_CONFIG[DOCKER]=${BUILD_CONFIG[DOCKER]:-""} + BUILD_CONFIG[CONTAINER_AS_ROOT]=${BUILD_CONFIG[CONTAINER_AS_ROOT]:-""} BUILD_CONFIG[TMP_SPACE_BUILD]=${BUILD_CONFIG[TMP_SPACE_BUILD]:-false} diff --git a/signalhandler.sh b/signalhandler.sh index 09df0ec33..d7a2c7c02 100755 --- a/signalhandler.sh +++ b/signalhandler.sh @@ -14,7 +14,7 @@ exit_script() { if [[ -z "${BUILD_CONFIG[KEEP_CONTAINER]}" ]] ; then - "${BUILD_CONFIG[USE_DOCKER]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[USE_DOCKER]}" rm -f {} + "${BUILD_CONFIG[CONTAINER_COMMAND]}" ps -a | awk '{ print $1,$2 }' | grep "${BUILD_CONFIG[CONTAINER_NAME]}" | awk '{print $1 }' | xargs -I {} "${BUILD_CONFIG[CONTAINER_COMMAND]}" rm -f {} fi echo "Process exited" trap - SIGINT SIGTERM # clear the trap From 10e228d349dfe05808b2d625786738cfd3de6daa Mon Sep 17 00:00:00 2001 From: Jiri Vanek Date: Tue, 25 Jun 2024 11:43:36 +0200 Subject: [PATCH 30/30] Improved warning about --custom-cacerts --- makejdk-any-platform.1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makejdk-any-platform.1 b/makejdk-any-platform.1 index 118702b6b..80cb4158c 100755 --- a/makejdk-any-platform.1 +++ b/makejdk-any-platform.1 @@ -99,7 +99,8 @@ This is typically used in conjunction with \fB<-T>\fR to create a custom path .TP .BR \-D ", " \-\-docker ", " \-\-podman build OpenJDK in a docker/podman container. -D will autodetect, using podman if found, docker otherwise. -When podman is used, there is bug requiring \fI--custom-cacerts\fR to be used. It is known issue: https://github.com/adoptium/temurin-build/issues/3862 +When podman is used, there is bug requiring \fI--custom-cacerts\fR to be used under some circumstances. +It is known issue: https://github.com/adoptium/temurin-build/issues/3862 .TP .BR \-\-debug-docker debug OpenJDK build script in a docker container. Only valid if \fB-D\fR is selected.