From 617aa7004bab715dd81737854dbfa717b99bc79f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 10 Nov 2023 17:45:52 +0100 Subject: [PATCH 1/5] run.py: reformat docstring It currently passes format checks only by abuse, and won't pass any more with following commits. Signed-off-by: Yann Dirson --- run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 30082d0..210c6ff 100755 --- a/run.py +++ b/run.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" Thin wrapper around "docker run" which simplifies the creation of a build environment for XCP-ng packages. """ +""" +Thin wrapper around "docker run". + +Simplifies the creation of a build environment for XCP-ng packages. +""" import argparse import os From 973e4ea1dfa3f1b8ee7c5c0439460fc849966834 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 17 Nov 2023 11:46:12 +0100 Subject: [PATCH 2/5] README: reduce the explicit mentions of Docker We're going to introduce podman support, this prepares the doc to make real changes easier to review. Signed-off-by: Yann Dirson --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7bbf0d1..e6bd3fd 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # xcp-ng-build-env -This docker config and collection of supporting scripts allows for creating -a docker container to work on and build a XCP-ng package from an SRPM or from -a directory containing a `SOURCES/` and a `SPECS/` directory along with appropriate -RPM spec file and software sources. -It will build a Docker container with the right build environment (including some +This container config and collection of supporting scripts allows for +creating a container to work on and build a XCP-ng package from an +SRPM or from a directory containing a `SOURCES/` and a `SPECS/` +directory along with appropriate RPM spec file and software sources. + +It will build a container with the right build environment (including some useful tools). Depending on the parameters, it will either do everything automatically to build a given package, or just install build-dependencies and let you work manually from a shell @@ -16,11 +17,12 @@ want. You'll need to install docker. Follow the instructions for your platform on https://www.docker.com/ -## Building the docker image(s) +## Building the container image(s) -You need one docker image per target version of XCP-ng. +You need one container image per target version of XCP-ng. -Clone this repository (outside any docker container), then use `build.sh` to generate the docker images for the wanted releases of XCP-ng. +Clone this repository (outside any container), then use `build.sh` to +generate the images for the wanted releases of XCP-ng. ``` Usage: ./build.sh {version_of_XCP_ng} @@ -115,7 +117,7 @@ git clone https://github.com/xcp-ng-rpms/xapi.git * `-b` / `--branch` allows to select which version of XCP-ng to work on (defaults to the latest known version if not specified). * `--no-exit` drops you to a shell after the build, instead of closing the container. Useful if the build fails and you need to debug. -* `--rm` destroys the container on exit. Helps preventing docker from using too much space on disk. You can still reclaim space afterwards by running `docker container prune` and `docker image prune` +* `--rm` destroys the container on exit. Helps preventing containers from using too much space on disk. You can still reclaim space afterwards by running `docker container prune` and `docker image prune` * `-v` / `--volume` (see *Mounting repos from outside the container* below) @@ -149,9 +151,7 @@ make If you'd like to develop using the tools on your host and preserve the changes to source and revision control but still use the container for building, you -can do using by using a docker volume. - -Once you have built your image you can run it with an extra argument to mount +can do using by mounting a volume in the container, using the `-v` option to mount a directory from your host to a suitable point inside the container. For example, if I clone some repos into a directory on my host, say `/work/code/`, then I can mount it inside the container as follows: From 7eb94095e8e9a95dae47c3869c0727fdf7d547ac Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 10 Nov 2023 14:05:31 +0100 Subject: [PATCH 3/5] Allow using podman as alternative to docker podman does not interact with a daemon running as root, so unlike docker it will not be able to modify files owned by arbitrary users including root. To keep with current practices, we need to pass --userns=keep-id so it can write into the user's directories: the `builder` user inside the container is indeed equivalent to the user launching the container. All other users including root are mapped UIDs not existing on the host (typically 100000 and above), so any file created by such users in build trees in the container will (by design) have those (numeric-only) owners on the host if they remain after the build. Signed-off-by: Yann Dirson --- README.md | 12 +++++++++--- build.sh | 4 +++- run.py | 12 ++++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e6bd3fd..22a9d3f 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,13 @@ want. ## Configuration -You'll need to install docker. Follow the instructions for your platform on -https://www.docker.com/ +You'll need to install docker or podman. Podman should be available +from your distro repositories, for Docker follow the instructions for +your platform on https://www.docker.com/ + +If you have both installed, docker will be used by default. If you +want to use a specific container runtime, set `XCPNG_OCI_RUNNER` to +the docker-compatible command to use (typically `podman` or `docker`). ## Building the container image(s) @@ -23,6 +28,7 @@ You need one container image per target version of XCP-ng. Clone this repository (outside any container), then use `build.sh` to generate the images for the wanted releases of XCP-ng. +Note that Docker and Podman store container images separately. ``` Usage: ./build.sh {version_of_XCP_ng} @@ -151,7 +157,7 @@ make If you'd like to develop using the tools on your host and preserve the changes to source and revision control but still use the container for building, you -can do using by mounting a volume in the container, using the `-v` option to mount +can do using by mouning a volume in the container, using the `-v` option to mount a directory from your host to a suitable point inside the container. For example, if I clone some repos into a directory on my host, say `/work/code/`, then I can mount it inside the container as follows: diff --git a/build.sh b/build.sh index 9c6ed1e..99d2a82 100755 --- a/build.sh +++ b/build.sh @@ -8,6 +8,8 @@ if [ -z "$1" ]; then exit fi +RUNNER="${XCPNG_OCI_RUNNER:-docker}" + cd $(dirname "$0") CUSTOM_ARGS=() @@ -56,7 +58,7 @@ fi CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_UID=${CUSTOM_UID}" ) CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_GID=${CUSTOM_GID}" ) -docker build \ +"$RUNNER" build \ "${CUSTOM_ARGS[@]}" \ -t xcp-ng/xcp-ng-build-env:${1} \ --ulimit nofile=1024 \ diff --git a/run.py b/run.py index 210c6ff..71fd24d 100755 --- a/run.py +++ b/run.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -Thin wrapper around "docker run". +Thin wrapper around "docker run" or "podman run". Simplifies the creation of a build environment for XCP-ng packages. """ @@ -20,6 +20,8 @@ DEFAULT_BRANCH = '8.3' DEFAULT_ULIMIT_NOFILE = 1024 +RUNNER = os.getenv("XCPNG_OCI_RUNNER", "docker") + def make_mount_dir(): """ Make a randomly-named directory under SRPMS_MOUNT_ROOT. """ srpm_mount_dir = os.path.join(SRPMS_MOUNT_ROOT, str(uuid.uuid4())) @@ -36,6 +38,10 @@ def copy_srpms(srpm_mount_dir, srpms): srpm_name = os.path.basename(srpm) shutil.copyfile(srpm, os.path.join(srpm_mount_dir, srpm_name)) +def is_podman(runner): + if os.path.basename(runner) == "podman": + return True + return False def main(): """ Main entry point. """ @@ -97,7 +103,9 @@ def main(): args = parser.parse_args(sys.argv[1:]) - docker_args = ["docker", "run", "-i", "-t", "-u", "builder"] + docker_args = [RUNNER, "run", "-i", "-t", "-u", "builder"] + if is_podman(RUNNER): + docker_args += ["--userns=keep-id"] if os.uname()[4] != "x86_64": docker_args += ["--platform", "linux/amd64"] if args.rm: From 16b6d7ae56f06b299b52c8e26a42892fd10db114 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 13 Nov 2023 16:30:09 +0100 Subject: [PATCH 4/5] run.py: identify podman hiding behind podman-docker `podman-docker` installs a `docker` binary calling `podman`, we have to call it to discover how to properly handle it. Signed-off-by: Yann Dirson --- run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run.py b/run.py index 71fd24d..20f2178 100755 --- a/run.py +++ b/run.py @@ -41,6 +41,8 @@ def copy_srpms(srpm_mount_dir, srpms): def is_podman(runner): if os.path.basename(runner) == "podman": return True + if subprocess.getoutput(f"{runner} --version").startswith("podman "): + return True return False def main(): From 7b6988b94b3c298c223787b2a7132cfcc198c42b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 10 Nov 2023 16:21:44 +0100 Subject: [PATCH 5/5] Fallback to podman automatically if docker is not installed Signed-off-by: Yann Dirson --- build.sh | 17 ++++++++++++++++- run.py | 10 +++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 99d2a82..1c6b21d 100755 --- a/build.sh +++ b/build.sh @@ -8,7 +8,22 @@ if [ -z "$1" ]; then exit fi -RUNNER="${XCPNG_OCI_RUNNER:-docker}" +RUNNER="" +if [ -n "$XCPNG_OCI_RUNNER" ]; then + RUNNER="$XCPNG_OCI_RUNNER" +else + SUPPORTED_RUNNERS="docker podman" + for COMMAND in $SUPPORTED_RUNNERS; do + if command -v $COMMAND >/dev/null; then + RUNNER="$COMMAND" + break + fi + done + if [ -z "$RUNNER" ]; then + echo >&2 "cannot find a supported runner: $SUPPORTED_RUNNERS" + exit 1 + fi +fi cd $(dirname "$0") diff --git a/run.py b/run.py index 20f2178..80b6011 100755 --- a/run.py +++ b/run.py @@ -20,7 +20,15 @@ DEFAULT_BRANCH = '8.3' DEFAULT_ULIMIT_NOFILE = 1024 -RUNNER = os.getenv("XCPNG_OCI_RUNNER", "docker") +RUNNER = os.getenv("XCPNG_OCI_RUNNER") +if RUNNER is None: + SUPPORTED_RUNNERS = "docker podman" + for command in SUPPORTED_RUNNERS.split(): + if shutil.which(command): + RUNNER = command + break + else: + raise Exception(f"cannot find a supported runner: {SUPPORTED_RUNNERS}") def make_mount_dir(): """ Make a randomly-named directory under SRPMS_MOUNT_ROOT. """