From 4b043b68e3fa290f6329770efa6d339d54bb095d Mon Sep 17 00:00:00 2001 From: Arun Ajith S Date: Sun, 21 Jul 2024 08:41:53 +0000 Subject: [PATCH 1/5] Reorganize eext base image boostrap code Separate out bootstrap code in barney.yaml to three parts in three different scripts: 1. Extract(boostrap/extract/extract.bash): Extracting bootstrap image from the tarball. 2. Setup dnf configuration(bootstrap/eext-repos/generate.bash): dnf configuration for both base image bootstrap and for install within base-image is done by this stage. 3. Install rpms(bootstrap/install-rpms/install-rpms.bash): The old install-rpms script is now reading rpms to be installed from the source files rpms-common and rpms- in the bootstrap/install-rpms directory. Also added a base-image-devel ref to be used by developers for interactive workflow. --- barney.yaml | 178 +++++++++++------- bootstrap/README.md | 53 +++++- bootstrap/eext-repos-build.repo.template | 16 -- .../eext-repos/eext-repos-build.repo.template | 16 ++ .../eext-repos/eext-repos-devel.repo.template | 37 ++++ bootstrap/eext-repos/eext-repos.spec | 75 ++++++++ bootstrap/eext-repos/generate-repo-file.bash | 35 ++++ bootstrap/eext-repos/generate.bash | 78 ++++++++ bootstrap/eext-repos/repos-build.env | 3 + bootstrap/eext-repos/repos-common.env | 10 + bootstrap/eext-repos/repos-devel.env | 5 + bootstrap/{ => extract}/CHECKSUM | 0 bootstrap/extract/extract.bash | 52 +++++ bootstrap/install-rpms | 7 - bootstrap/install-rpms/install-rpms.bash | 59 ++++++ bootstrap/install-rpms/rpms-build | 4 + bootstrap/install-rpms/rpms-common | 15 ++ bootstrap/install-rpms/rpms-devel | 5 + bootstrap/install-rpms/rpms-test | 4 + 19 files changed, 557 insertions(+), 95 deletions(-) delete mode 100644 bootstrap/eext-repos-build.repo.template create mode 100644 bootstrap/eext-repos/eext-repos-build.repo.template create mode 100644 bootstrap/eext-repos/eext-repos-devel.repo.template create mode 100644 bootstrap/eext-repos/eext-repos.spec create mode 100755 bootstrap/eext-repos/generate-repo-file.bash create mode 100755 bootstrap/eext-repos/generate.bash create mode 100644 bootstrap/eext-repos/repos-build.env create mode 100644 bootstrap/eext-repos/repos-common.env create mode 100644 bootstrap/eext-repos/repos-devel.env rename bootstrap/{ => extract}/CHECKSUM (100%) create mode 100755 bootstrap/extract/extract.bash delete mode 100644 bootstrap/install-rpms create mode 100755 bootstrap/install-rpms/install-rpms.bash create mode 100644 bootstrap/install-rpms/rpms-build create mode 100644 bootstrap/install-rpms/rpms-common create mode 100644 bootstrap/install-rpms/rpms-devel create mode 100644 bootstrap/install-rpms/rpms-test diff --git a/barney.yaml b/barney.yaml index 77302a0..0c8a467 100644 --- a/barney.yaml +++ b/barney.yaml @@ -5,29 +5,22 @@ generators: images: - internal/bootstrap/iso-extraction-floor: + internal/bootstrap/extract-floor: entry: mutables: - /var/work units: - image: barney.ci/alpine%pkg/alpine-base - - image: barney.ci/alpine%pkg/gettext + - image: barney.ci/alpine%pkg/bash - image: barney.ci/alpine%pkg/wget - image: barney.ci/alpine%network - image: barney.ci/alpine%apk-finalizers - build: | mkdir -p /dest/var/work + cp -a bootstrap/extract /dest - internal/bootstrap/src-deps: + internal/bootstrap/extract: no-create-mountpoints: true - units: - - build: | - cp -a bootstrap/* /dest/ - cp -a pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /dest/ - cp -a pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem /dest/ - rm -f /dest/README* - - internal/bootstrap: description: | Minimal bootstrapping environment used to bootstrap almalinux. @@ -36,11 +29,7 @@ images: is readily available on the mirror. This is seldom changed and any change here will update all eext snapshots. - We wipe out the dnf/yum repo configuration in the tarball, - and setup our own curated set of frozen almalinux repos. - - We also include an install-rpms script which is used to bootstrap - the almalinux image. + We wipe out the dnf/yum repo configuration in the tarball. entry: share-net: true mounts: @@ -51,65 +40,105 @@ images: - /var/cache - /var/lib/dnf units: - - floor: .%internal/bootstrap/iso-extraction-floor + - floor: .%internal/bootstrap/extract-floor entry: env: DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory} - BOOTSTRAP_PATH: eext-sources/bootstrap/CentOS-Stream - BOOTSTRAP_FILE: CentOS-Stream-Container-Base-9-20240715.0 - BOOTSTRAP_EXTENSION: tar.xz - DNF_DISTRO_REPO: eext-alma-vault - DNF_DISTRO_REPO_VERSION: "9.3" - DNF_EPEL_REPO: eext-snapshots-local/epel9 - DNF_EPEL_REPO_SNAPSHOT_VERSION: v20240127-1 - DNF_EPEL_REPO_DISTRO_VERSION: "9" sources: [] - mappings: - /src/bootstrap: .%internal/bootstrap/src-deps build: | - # URL of tarball with OS image - export ARCH=$(uname -m) - bootstrap_filename="${BOOTSTRAP_FILE}.${ARCH}.${BOOTSTRAP_EXTENSION}" - bootstrap_url="${DNF_HOST}/${BOOTSTRAP_PATH}/${bootstrap_filename}" + cd /var/work + /extract/extract.bash /extract + touch $DESTDIR/etc/resolv.conf - # Download the tarball into the mutable working dir + internal/bootstrap/eext-repos-floor: + description: | + Alpine floor with the eext-repos src directory mapped in. + gettext provides envsubst used by generate-repo-file.bash called from generate.bash + rpm-dev provides rpmbuild used by generate.bash. + entry: + mutables: + - /var/work + - /var/tmp + units: + - image: barney.ci/alpine%pkg/alpine-base + - image: barney.ci/alpine%pkg/bash + - image: barney.ci/alpine%pkg/gettext + - image: barney.ci/alpine%pkg/rpm-dev + - image: barney.ci/alpine%apk-finalizers + - build: | + mkdir -p /dest/var/work + cp -a bootstrap/eext-repos /dest + cp -a pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /dest/eext-repos + cp -a pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem /dest/eext-repos + + internal/bootstrap/eext-repos: + no-create-mountpoints: true + description: | + Setup curated frozen dnf repo list for build reproducibility. + There're two files, eext-repo-build.repo and eext-repo-devel.repo. These files are + installed in /eext-repos directory. + We also install the eext-repo-build.repo in /etc/yum.repos.d to be used for base image + bootstrap. + + We also build an rpm each with the contents being the repo file, + and instructions to install at /etc/yum.repos.d. + The RPMs are named eext-repos-build and eext-repos-devel with approiate architecture + and .rpm extensions. They are installed in /RPMS. + units: + - floor: .%internal/bootstrap/eext-repos-floor + entry: + env: + DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory} + sources: [] + build: | cd /var/work - wget ${bootstrap_url} - - # Validate downloaded tarball - grep "${bootstrap_filename}" /src/bootstrap/CHECKSUM | sha256sum -wc - - # Extract tarball and setup rootfs - # This is a nested tarball, the real rootfs is in layer.tar - # Extract the firt level tarball inside the extr subdirectory - # within the working directory and and then extract the - # second level layer.tar directly to /dest - mkdir extr - tar --strip-components=1 -C ./extr -xf ./${bootstrap_filename} - tar -xf ./extr/layer.tar -C /dest - - # Now modify the extracted file system to remove unwanted - # stuff and add extra stuff. - - # Remove any pre-configured yum repos - rm /dest/etc/yum.repos.d/* - # Setup install-rpms shell script - mkdir -p /dest/usr/bin - chmod 555 /dest/usr/bin - cp -a /src/bootstrap/install-rpms /dest/usr/bin/ - chmod 0755 /dest/usr/bin/install-rpms - # Setup gpg keys - mkdir -p /dest/usr/share/distribution-gpg-keys - chmod 555 /dest/usr/share/distribution-gpg-keys - cp -a /src/bootstrap/*.pem /dest/usr/share/distribution-gpg-keys/ - # Setup curated frozen dnf repo list for build reproducibility - envsubst < /src/bootstrap/eext-repos-build.repo.template > /dest/etc/yum.repos.d/eext-repos-build.repo - - base-image: + /eext-repos/generate.bash /eext-repos + + internal/bootstrap/install-rpms: + no-create-mountpoints: true + description: | + Setup install-rpm scripts and collaterals for bootstrapping + units: + - build: | + cp -a bootstrap/install-rpms /dest + + internal/bootstrap/dnf-dirs: + no-create-mountpoints: true + description: | + Setup empty directories needed by dnf + units: + - sources: [] + build: | + mkdir -p /dest/var/lib/dnf + chmod 755 /dest/var/lib/dnf + + internal/bootstrap: + description: | + Final bootstrap layer used as floor to build base images. + units: + - image: .%internal/bootstrap/dnf-dirs + - image: .%internal/bootstrap/extract + - image: .%internal/bootstrap/eext-repos + - image: .%internal/bootstrap/install-rpms + + base-image-build: + units: + - floor: .%internal/bootstrap + sources: [] + build: | + echo "install rpms" + /install-rpms/install-rpms.bash \ + --common-rpms-file /install-rpms/rpms-common \ + --extra-rpms-file /install-rpms/rpms-build + + base-image-devel: units: - floor: .%internal/bootstrap sources: [] - build: install-rpms autoconf automake coreutils git rpm rpmdevtools rpm-build make mock python3-devel quilt + build: | + /install-rpms/install-rpms.bash \ + --common-rpms-file /install-rpms/rpms-common \ + --extra-rpms-file /install-rpms/rpms-devel go-binaries: description: | @@ -124,10 +153,8 @@ images: cp -a /src/static/usr/bin /dest/usr/bin chmod 555 /dest/usr/bin - eext: + eext-configfiles: units: - - image: .%base-image - - image: .%go-binaries - build: | mkdir -p $DESTDIR/usr/share/eext cp -a ./configfiles/* $DESTDIR/usr/share/eext @@ -135,6 +162,16 @@ images: cp -a ./pki/* $DESTDIR/etc/pki/eext mkdir -p $DESTDIR/etc touch $DESTDIR/etc/resolv.conf + + eext: + units: + - image: .%base-image-build + - image: .%go-binaries + - image: .%eext-configfiles + - build: | + mkdir -p $DESTDIR/etc + touch $DESTDIR/etc/resolv.conf + entry: mutables: &eext-mutables - /var/eext @@ -159,8 +196,11 @@ images: - floor: .%internal/bootstrap sources: [] build: | - install-rpms autoconf automake coreutils golang git rpm rpmdevtools rpm-build make mock python3-devel quilt + /install-rpms/install-rpms.bash \ + --common-rpms-file /install-rpms/rpms-common \ + --extra-rpms-file /install-rpms/rpms-devel touch $DESTDIR/etc/resolv.conf + entry: env: GOCACHE: /tmp/gocache diff --git a/bootstrap/README.md b/bootstrap/README.md index dc3fa8a..01a83c1 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -1,4 +1,17 @@ -# Uploading Bootstrap Tarball +# Bootstrapping a Redhat based distro image for the eext tool + +This document explains how a base image for eext buids or development workflow is built. The base image should contain the utilties called by the eext static go binary. +The main dependencies are rpm, rpmbuild and mock. +We support two base images, one for running eext builds in Abuilds/bsy builds, and another for running eext builds in an user's interactive development environment. We call them base-image-build and base-image-devel. + + +## Bootstrap image +We need a RPM based rootfs to boostrap the eext base image which is today based of AlmaLinux. +We construct this rootfs image form a tarball published by some RPM based distro. +We need a tarball per target-arch, ie x86_64 amd aarch64. Note that i686 RPMs are built with the x86_64 eext base image itself. +This bootstrap tarball need not be AlmaLinux, it can be any RPM based distro. We're using a CentOS Stream tarball for this purpose, because they publish it to the mirror with checksums. + +### Updating/Uploading the bootstrap tarball 1. Download a CentOS-Stream-Container-Base tarball with a timestamp in its name, like `20230704`, from: - [https://cloud.centos.org/centos/9-stream/x86_64/images/](https://cloud.centos.org/centos/9-stream/x86_64/images/) @@ -7,7 +20,41 @@ ``` curl -H "Authorization: Bearer ${ARTIFACTORY_TOKEN}" -X PUT https://artifactory.infra.corp.arista.io/artifactory/eext-sources/bootstrap/CentOS-Stream/ -T ``` -3. Update the `CHECKSUM` file in the local repo for the new entries from the `CHECKSUM` files: +3. Update the `extract/CHECKSUM` file in the local repo for the new entries from the `CHECKSUM` files: - [https://cloud.centos.org/centos/9-stream/x86_64/images/CHECKSUM](https://cloud.centos.org/centos/9-stream/x86_64/images/CHECKSUM) - [https://cloud.centos.org/centos/9-stream/aarch64/images/CHECKSUM](https://cloud.centos.org/centos/9-stream/aarch64/images/CHECKSUM) -4. Update the `EEXT_BOOTSTRAP_VERSION` environment variable in `barney.yaml`. +4. Update the `bootstrap_filename_version` variable in `extract/extract.bash`. + +## Base Image Build + +### Repo configuration +To build an AlmaLinux base-image we run `dnf --installroot` inside the bootstrap image. We maintain our own curated dnf configuration on the `eext-repos` directory. +The configuration points to the second most recent dot release of the main AlmaLinux that eext is tracking. The idea is that this vaulted and frozen, giving us reproducible builds. +Note that we make them point to a local artifactory remote-repo/mirror of the upstream AlmaLinux repo. This makes sure we don't have unnecessary internet traffic, and we have a copy of the dependencies maintained in the remote repo's cache. + +We maintain two such configuration file templates in our source for each variant of the base image: +1 `eext-repos/eext-repos-build.repo.template` +2.`eext-repos/eext-repos-devel.repo.template` + +The build specific configuration just includes the basic repos needed for satisfying the the eext dependencies like `rpm` and `mock`. +The development configuration configures a superset of the build configuration, because the developer might need to install more tools like editors etc. +We try to track the major/minor versions of the distros similarly between the two. The `eext-repos/repos-common.env` file holds the versions, actual repo names, URLs etc as environment variables. + +We want the build specific repos to point to the `eext-alma-vault` artifactory remote repo, while we point the development repos to the the global `alma-vault` remote repository. +This makes sure that the dependencies in the developer's environment don't pollute `eext-alma-vault` cache. +The base-image specific configuration exists in the `eext-repos/repos-build.env` and `eext-repos/repos-devel.env` files. + +The `eext-repos/generate.bash` script loads the `.env` files and builds the actual repo configuration files from the templates. + +Note that we need repo configuration in two stages: +1. For the boostrap container for base-image generation +2. Repo configuration in the base image itself. + +Note that the distinction in the repo configuration exists only for `2`. The bootstrap container is always configured with the build variant. +The developer just uses these repos to further install further packages in his bus or docker container with dnf. + +### Base Image Contents +Once the bootstrap image and repo configuration is ready, we run the `install-rpms/install-rpms.bash` script to build the base image. +The set of rpms to be installed is specified in the text files: `install-rpms/rpms-common`, `install-rpms/rpms-build` and `install-rpms/rpms-devel`. +The `install-rpms` script takes two arguments, `--common-rpms-file rpms-common` and `--extra-rpms-file (rpms-build | rpms-devel)`. + diff --git a/bootstrap/eext-repos-build.repo.template b/bootstrap/eext-repos-build.repo.template deleted file mode 100644 index fde11aa..0000000 --- a/bootstrap/eext-repos-build.repo.template +++ /dev/null @@ -1,16 +0,0 @@ -[BaseOS] -baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/BaseOS/${ARCH}/os/ -enabled=1 -gpgkey=file:///usr/share/distribution-gpg-keys/alma9-b86b3716-gpg-pubkey.pem - -[AppStream] -baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/AppStream/${ARCH}/os/ -exclude=podman -enabled=1 -gpgkey=file:///usr/share/distribution-gpg-keys/alma9-b86b3716-gpg-pubkey.pem - -[epel9-snapshot] -baseurl=${DNF_HOST}/${DNF_EPEL_REPO}/${DNF_EPEL_REPO_SNAPSHOT_VERSION}/${DNF_EPEL_REPO_DISTRO_VERSION}/Everything/${ARCH}/ -enabled=1 -gpgkey=file:///usr/share/distribution-gpg-keys/epel9-3228467c-gpg-pubkey.pem - diff --git a/bootstrap/eext-repos/eext-repos-build.repo.template b/bootstrap/eext-repos/eext-repos-build.repo.template new file mode 100644 index 0000000..64ad096 --- /dev/null +++ b/bootstrap/eext-repos/eext-repos-build.repo.template @@ -0,0 +1,16 @@ +[baseos] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/BaseOS/${ARCH}/os/ +enabled=1 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[appstream] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/AppStream/${ARCH}/os/ +exclude=podman +enabled=1 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[epel9-snapshot] +baseurl=${DNF_HOST}/${DNF_EPEL_SNAPSHOT_REPO}/${DNF_EPEL_SNAPSHOT_REPO_VERSION}/${DNF_EPEL_DISTRO_VERSION}/Everything/${ARCH}/ +enabled=1 +gpgkey=file:///usr/share/eext-gpg-keys/epel9-3228467c-gpg-pubkey.pem + diff --git a/bootstrap/eext-repos/eext-repos-devel.repo.template b/bootstrap/eext-repos/eext-repos-devel.repo.template new file mode 100644 index 0000000..1c127fd --- /dev/null +++ b/bootstrap/eext-repos/eext-repos-devel.repo.template @@ -0,0 +1,37 @@ +[baseos] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/BaseOS/${ARCH}/os/ +enabled=1 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[appstream] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/AppStream/${ARCH}/os/ +exclude=podman +enabled=1 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[epel9-snapshot] +baseurl=${DNF_HOST}/${DNF_EPEL_SNAPSHOT_REPO}/${DNF_EPEL_SNAPSHOT_REPO_VERSION}/${DNF_EPEL_DISTRO_VERSION}/Everything/${ARCH}/ +enabled=1 +gpgkey=file:///usr/share/eext-gpg-keys/epel9-3228467c-gpg-pubkey.pem + +[crb] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/CRB/${ARCH}/os/ +enabled=0 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[devel] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/devel/${ARCH}/os/ +enabled=0 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[extras] +baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/extras/${ARCH}/os/ +enabled=0 +gpgkey=file:///usr/share/eext-gpg-keys/alma9-b86b3716-gpg-pubkey.pem + +[epel] +baseurl=${DNF_HOST}/${DNF_EPEL_REPO}/${ARCH}/ +enabled=0 +exclude=mock +gpgkey=file:///usr/share/eext-gpg-keys/epel9-3228467c-gpg-pubkey.pem + diff --git a/bootstrap/eext-repos/eext-repos.spec b/bootstrap/eext-repos/eext-repos.spec new file mode 100644 index 0000000..d32089a --- /dev/null +++ b/bootstrap/eext-repos/eext-repos.spec @@ -0,0 +1,75 @@ +Name: eext-release +Version: %{eext_alma_version} +Release: alma_%{eext_alma_release}.eext_1 +Summary: Empty AlmaLinux release for eext +License: GPLv2 + +Source0: eext-repos-build.repo +Source1: eext-repos-devel.repo +Source2: alma9-b86b3716-gpg-pubkey.pem +Source3: alma9-b86b3716-gpg-pubkey.pem + +%description +Dummy package to define srpm + +%package -n eext-gpg-keys +Summary: gpgkeys for curated repos +Requires: almalinux-release = %{eext_alma_version}-%{eext_alma_release} + +%description -n eext-gpg-keys +gpgkeys for curated repos + +%package -n eext-repos-build +Summary: Subset of vaulted almalinux repos to be used by eext build +Requires: almalinux-release = %{eext_alma_version}-%{eext_alma_release} +# Remove any almalinux-repos +Obsoletes: almalinux-repos = %{eext_alma_version}-%{eext_alma_release} +Provides: almalinux-repos = %{eext_alma_version}-%{eext_alma_release} +# Don't allow epel repos to be configured +Conflicts: epel-release +Requires: eext-gpg-keys = %{version}-%{release} + +%description -n eext-repos-build +Subset of vaulted almalinux repos to be used by eext build. +The vaulted penultimate dot release is used to ensure a frozen dnf repo for build reproducibility. +The "eext-alma-vault" local mirror is used. + +%package -n eext-repos-devel +Summary: Vaulted almalinux repos and other disabled repos to be used for eext dev workflow. +Requires: almalinux-release = %{eext_alma_version}-%{eext_alma_release} +# Remove any almalinux-repos +Obsoletes: almalinux-repos = %{eext_alma_version}-%{eext_alma_release} +Obsoletes: eext-repos-build = %{version}-%{release} +Provides: almalinux-repos = %{eext_alma_version}-%{eext_alma_release} +# Don't allow epel repos to be configured +Conflicts: epel-release +Requires: eext-gpg-keys = %{version}-%{release} + +%description -n eext-repos-devel +Vaulted almalinux repos and other disabled repos to be used for eext dev workflow. +The "alma-vault" local mirror/cache is used to avoid polluting the "eext-alma-vault" +local mirror's cache with RPMs pulled in by developers for their environment. +This is because the local mirror's cache can be snapshoted for every release to hold +the dependency set. + +%install +# create /etc/yum.repos.d +install -d -m 0755 %{buildroot}%{_sysconfdir}/yum.repos.d +install -p -m 0644 %{SOURCE0} %{buildroot}%{_sysconfdir}/yum.repos.d/ +install -p -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/yum.repos.d/ +mkdir -p %{buildroot}%{_datadir}/eext-gpg-keys +cp -a %{SOURCE2} %{buildroot}%{_datadir}/eext-gpg-keys +cp -a %{SOURCE3} %{buildroot}%{_datadir}/eext-gpg-keys + +%files -n eext-gpg-keys +%{_datadir}/eext-gpg-keys + +%files -n eext-repos-build +%{_sysconfdir}/yum.repos.d/eext-repos-build.repo + +%files -n eext-repos-devel +%{_sysconfdir}/yum.repos.d/eext-repos-devel.repo + +%changelog +* Sun Jul 21 2024 Arun Ajith S - 9.3-alma_1.el9.eext_1 +- Creating spec file when we're based off almalinux-release-9.3-1 diff --git a/bootstrap/eext-repos/generate-repo-file.bash b/bootstrap/eext-repos/generate-repo-file.bash new file mode 100755 index 0000000..72109af --- /dev/null +++ b/bootstrap/eext-repos/generate-repo-file.bash @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e +set -x + +usage() { + echo "Usage: $0 " + exit 1 +} + +if [ $# -ne 3 ]; then + usage +fi + +TEMPLATE_FILE=$1 +ENV_FILE=$2 +OUTPUT_FILE=$3 + +if [ ! -f "$TEMPLATE_FILE" ]; then + echo "Error: Template file '$TEMPLATE_FILE' not found." + exit 1 +fi + +if [ ! -f "$ENV_FILE" ]; then + echo "Error: Environment file '$ENV_FILE' not found." + exit 1 +fi + +# Load environment variables from the environment file +set -a +source "$ENV_FILE" +set +a + +envsubst < "$TEMPLATE_FILE" > "$OUTPUT_FILE" + diff --git a/bootstrap/eext-repos/generate.bash b/bootstrap/eext-repos/generate.bash new file mode 100755 index 0000000..6c9d1c5 --- /dev/null +++ b/bootstrap/eext-repos/generate.bash @@ -0,0 +1,78 @@ +#!/bin/bash + +set -e +set -x + +generate_repo_files() { + WORKING_DIR=`pwd` + pushd ${COLLATERALS_DIR} + ./generate-repo-file.bash ./eext-repos-build.repo.template ./repos-build.env "${WORKING_DIR}/eext-repos-build.repo" + ./generate-repo-file.bash ./eext-repos-devel.repo.template ./repos-devel.env "${WORKING_DIR}/eext-repos-devel.repo" + popd + mkdir -p /dest/etc/yum.repos.d + chmod 755 /dest/etc/yum.repos.d + cp -a ./eext-repos-build.repo /dest/etc/yum.repos.d +} + +generate_rpm() { + WORKING_DIR=`pwd` + mkdir -p rpmbuild + pushd rpmbuild + mkdir SOURCES SPECS + + cp "${COLLATERALS_DIR}"/eext-repos.spec SPECS/ + cp "${WORKING_DIR}"/*.repo "${COLLATERALS_DIR}"/*.pem SOURCES/ + rpmbuild --define "_topdir `pwd`" \ + --define "eext_alma_version ${DNF_DISTRO_REPO_VERSION}" \ + --define "eext_alma_release ${DNF_DISTRO_REPO_RELEASE}" \ + --define "source_date_epoch_from_changelog 1" \ + --define "use_source_date_epoch_as_buildtime 1" \ + --define "clamp_mtime_to_source_date_epoch 1" \ + --define "_buildhost eext-buildhost" \ + --define "_build_name_fmt %%{NAME}.rpm" \ + -ba ./SPECS/eext-repos.spec + + if [ ! -f "./RPMS/eext-repos-build.rpm" ]; then + echo "Error: './RPMS/ext-repos-build.rpm' not found after rpmbuild." + fi + + if [ ! -f "./RPMS/eext-repos-devel.rpm" ]; then + echo "Error: './RPMS/eext-repos-devel.rpm' not found after rpmbuild." + fi + + mkdir -p /dest/RPMS + chmod 755 /dest/RPMS + cp -a ./RPMS/*.rpm /dest/RPMS/ + popd +} + +setup_gpg_keys() { + mkdir -p /dest/usr/share/eext-gpg-keys + chmod 755 /dest/usr/share/eext-gpg-keys + cp "${COLLATERALS_DIR}"/*.pem /dest/usr/share/eext-gpg-keys/ +} + +usage() { + echo "Usage: $0 " + exit 1 +} + +if [ $# -ne 1 ]; then + usage +fi + +COLLATERALS_DIR=$1 +if [ ! -d "$COLLATERALS_DIR" ]; then + echo "Error: Collaterals directory '$COLLATERALS_DIR' not found." + exit 1 +fi + +TEMPLATE_FILE=$1 +set -a +source "$COLLATERALS_DIR/repos-common.env" +set +a +export ARCH=$(uname -m) + +generate_repo_files +generate_rpm +setup_gpg_keys diff --git a/bootstrap/eext-repos/repos-build.env b/bootstrap/eext-repos/repos-build.env new file mode 100644 index 0000000..b31ef31 --- /dev/null +++ b/bootstrap/eext-repos/repos-build.env @@ -0,0 +1,3 @@ +# The build specific eext-alma-vault remote repo is used. +DNF_DISTRO_REPO="eext-alma-vault" + diff --git a/bootstrap/eext-repos/repos-common.env b/bootstrap/eext-repos/repos-common.env new file mode 100644 index 0000000..d62049f --- /dev/null +++ b/bootstrap/eext-repos/repos-common.env @@ -0,0 +1,10 @@ +# We use the vaulted penumltimate dot release to ensure a frozen dnf repo. +# We use a artifactory remote repository hosted in the Arista network which +# is configured to mirror the upstream almalinux repos. +DNF_DISTRO_REPO_VERSION="9.3" +DNF_DISTRO_REPO_RELEASE="1.el9" +DNF_EPEL_SNAPSHOT_REPO="eext-snapshots-local/epel9" +DNF_EPEL_SNAPSHOT_REPO_VERSION="v20240127-1" +DNF_EPEL_DISTRO_VERSION="9" +ARCH=$(arch) + diff --git a/bootstrap/eext-repos/repos-devel.env b/bootstrap/eext-repos/repos-devel.env new file mode 100644 index 0000000..dd71417 --- /dev/null +++ b/bootstrap/eext-repos/repos-devel.env @@ -0,0 +1,5 @@ +# The alma-vault remote repo is used since we don't want to pollute the cache +# of the build-specific eext-alma-vault with dependencies installed in the +# development environment. +DNF_DISTRO_REPO="alma-vault" +DNF_EPEL_REPO="centos-9-epel" diff --git a/bootstrap/CHECKSUM b/bootstrap/extract/CHECKSUM similarity index 100% rename from bootstrap/CHECKSUM rename to bootstrap/extract/CHECKSUM diff --git a/bootstrap/extract/extract.bash b/bootstrap/extract/extract.bash new file mode 100755 index 0000000..75f5014 --- /dev/null +++ b/bootstrap/extract/extract.bash @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e +set -x + +usage() { + echo "Usage: $0 " + exit 1 +} + +if [ $# -ne 1 ]; then + usage +fi + +COLLATERALS_DIR=$1 +if [ ! -d "$COLLATERALS_DIR" ]; then + echo "Error: Collaterals directory '$COLLATERALS_DIR' not found." + exit 1 +fi + +arch=$(arch) +bootstrap_file_repodir_path="eext-sources/bootstrap/CentOS-Stream" +bootstrap_filename_base="CentOS-Stream-Container-Base-9" +bootstrap_filename_version="20240715.0" +bootstrap_filename_extension="tar.xz" +bootstrap_filename="${bootstrap_filename_base}-${bootstrap_filename_version}.${arch}.${bootstrap_filename_extension}" + +# URL of tarball with OS image +bootstrap_url="${DNF_HOST}/${bootstrap_file_repodir_path}/${bootstrap_filename}" + +# Download the tarball into the mutable working dir +wget ${bootstrap_url} + +# Validate downloaded tarball +grep "${bootstrap_filename}" "${COLLATERALS_DIR}/CHECKSUM" | sha256sum -wc + +# Extract tarball and setup rootfs +# This is a nested tarball, the real rootfs is in layer.tar +# Extract the first level tarball inside the extract subdirectory +# within the working directory and and then extract the +# second level layer.tar directly to /dest +mkdir extract +tar --strip-components=1 -C ./extract -xf ./${bootstrap_filename} +tar -xf ./extract/layer.tar -C /dest + +# Now modify the extracted file system to remove unwanted + +# Note that we'll layer on our own curated yum repos and gpg keys into the bootstrap +# image instead of using the one from the bootstrap image +rm -rf /dest/etc/yum.repos.d +rm -rf /dest/usr/share/distribution-gpg-keys + diff --git a/bootstrap/install-rpms b/bootstrap/install-rpms deleted file mode 100644 index 40c776a..0000000 --- a/bootstrap/install-rpms +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -dnf --assumeyes --installroot=/dest --noplugins \ - --config=/etc/dnf/dnf.conf \ - --setopt=cachedir=/var/cache/dnf \ - --setopt=reposdir=/etc/yum.repos.d \ - --setopt=varsdir=/etc/dnf \ - install "$@" diff --git a/bootstrap/install-rpms/install-rpms.bash b/bootstrap/install-rpms/install-rpms.bash new file mode 100755 index 0000000..20af5e1 --- /dev/null +++ b/bootstrap/install-rpms/install-rpms.bash @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e +set -x + +dnf_install() { + dnf --assumeyes --installroot=/dest --noplugins \ + --config=/etc/dnf/dnf.conf \ + --setopt=cachedir=/var/cache/dnf \ + --setopt=reposdir=/etc/yum.repos.d \ + --setopt=varsdir=/etc/dnf \ + install "$@" +} + +usage() { + echo "Usage: $0 --common-rpms-file FILE --extra-rpms-file FILE" + exit 1 +} + +# Parse command-line options +while [[ "$#" -gt 0 ]]; do + case $1 in + --common-rpms-file) + common_rpms_file="$2" + shift 2 + ;; + --extra-rpms-file) + extra_rpms_file="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +if [[ -z "$common_rpms_file" && -z "$extra_rpms_file" ]]; then + echo "Error: At least one of the options must be specified." + usage +fi + +rpms=() +for file in "$common_rpms_file" "$extra_rpms_file"; do + if [[ -n "$file" && ! -f "$file" ]]; then + echo "Error: File '$file' does not exist." + exit 1 + fi + + mapfile -t tmp_array < <(awk '!/^#/' "$file") + rpms+=("${tmp_array[@]}") +done + +if [[ ${#rpms[@]} -eq 0 ]]; then + echo "Error: No RPMs specified" + exit 1 +fi + +dnf_install "${rpms[@]}" + diff --git a/bootstrap/install-rpms/rpms-build b/bootstrap/install-rpms/rpms-build new file mode 100644 index 0000000..461c7b3 --- /dev/null +++ b/bootstrap/install-rpms/rpms-build @@ -0,0 +1,4 @@ +# Specify extra rpms to be installed in the build base-image. +# Use a path specification for local rpm, otherwise it'll be installed +# from one of the repos used for bootstrapping. +/RPMS/eext-repos-build.rpm diff --git a/bootstrap/install-rpms/rpms-common b/bootstrap/install-rpms/rpms-common new file mode 100644 index 0000000..b439703 --- /dev/null +++ b/bootstrap/install-rpms/rpms-common @@ -0,0 +1,15 @@ +# Specify common rpms to be installed in all base-images +# Use a path specification for local rpm, otherwise it'll be installed +# from one of the repos used for bootstrapping. +autoconf +automake +coreutils +git +make +mock +python3-devel +quilt +rpm +rpmdevtools +rpm-build +/RPMS/eext-gpg-keys.rpm diff --git a/bootstrap/install-rpms/rpms-devel b/bootstrap/install-rpms/rpms-devel new file mode 100644 index 0000000..73239b1 --- /dev/null +++ b/bootstrap/install-rpms/rpms-devel @@ -0,0 +1,5 @@ +# Specify extra rpms to be installed in the devel base-image +# Use a path specification for local rpm, otherwise it'll be installed +# from one of the repos used for bootstrapping. +/RPMS/eext-repos-devel.rpm +golang diff --git a/bootstrap/install-rpms/rpms-test b/bootstrap/install-rpms/rpms-test new file mode 100644 index 0000000..f82c801 --- /dev/null +++ b/bootstrap/install-rpms/rpms-test @@ -0,0 +1,4 @@ +# Specify extra rpms to be installed for the testfloor +# Use a path specification for local rpm, otherwise it'll be installed +# from one of the repos used for bootstrapping. +golang From 29c3e540998b18bd5db35308be361578c4e47d8e Mon Sep 17 00:00:00 2001 From: Arun Ajith S Date: Wed, 4 Sep 2024 10:37:36 +0000 Subject: [PATCH 2/5] Refactor the code to be more cache aware Using Snaipe's new bfg feature that allows file inlining to make the bootstrap, base image build and eext builds more bsy cache aware. Roll-With: barney.ci/barneyfile#203a8fea892dbd8d9058c7a9fce98d974f8ea99b --- barney.yaml | 153 +++++++++++++++++++++-------- bootstrap/eext-repos/generate.bash | 44 +++++++-- bootstrap/extract/extract.bash | 8 +- bootstrap/install-rpms/rpms-test | 4 - 4 files changed, 153 insertions(+), 56 deletions(-) delete mode 100644 bootstrap/install-rpms/rpms-test diff --git a/barney.yaml b/barney.yaml index 0c8a467..9ef4063 100644 --- a/barney.yaml +++ b/barney.yaml @@ -5,7 +5,15 @@ generators: images: - internal/bootstrap/extract-floor: + internal/bootstrap/extract/sources: + units: + - files: + - from: . + paths: + - 'bootstrap/extract/extract.bash' + - 'bootstrap/extract/CHECKSUM' + + internal/bootstrap/extract/floor: entry: mutables: - /var/work @@ -15,9 +23,7 @@ images: - image: barney.ci/alpine%pkg/wget - image: barney.ci/alpine%network - image: barney.ci/alpine%apk-finalizers - - build: | - mkdir -p /dest/var/work - cp -a bootstrap/extract /dest + - image: .%internal/bootstrap/extract/sources internal/bootstrap/extract: no-create-mountpoints: true @@ -40,17 +46,26 @@ images: - /var/cache - /var/lib/dnf units: - - floor: .%internal/bootstrap/extract-floor + - floor: .%internal/bootstrap/extract/floor entry: env: DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory} sources: [] build: | cd /var/work - /extract/extract.bash /extract + bash /bootstrap/extract/extract.bash /bootstrap/extract/CHECKSUM touch $DESTDIR/etc/resolv.conf - internal/bootstrap/eext-repos-floor: + internal/bootstrap/eext-repos/sources: + units: + - files: + - from: . + match: + - 'bootstrap/eext-repos/*' + - 'pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem' + - 'pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem' + + internal/bootstrap/eext-repos/floor: description: | Alpine floor with the eext-repos src directory mapped in. gettext provides envsubst used by generate-repo-file.bash called from generate.bash @@ -65,11 +80,10 @@ images: - image: barney.ci/alpine%pkg/gettext - image: barney.ci/alpine%pkg/rpm-dev - image: barney.ci/alpine%apk-finalizers - - build: | + - image: .%internal/bootstrap/eext-repos/sources + - sources: [] + build: | mkdir -p /dest/var/work - cp -a bootstrap/eext-repos /dest - cp -a pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /dest/eext-repos - cp -a pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem /dest/eext-repos internal/bootstrap/eext-repos: no-create-mountpoints: true @@ -85,22 +99,46 @@ images: The RPMs are named eext-repos-build and eext-repos-devel with approiate architecture and .rpm extensions. They are installed in /RPMS. units: - - floor: .%internal/bootstrap/eext-repos-floor + - floor: .%internal/bootstrap/eext-repos/floor entry: env: DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory} sources: [] build: | cd /var/work - /eext-repos/generate.bash /eext-repos + # First argument is collaterals dir, further args are pem files + bash /bootstrap/eext-repos/generate.bash /bootstrap/eext-repos /pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem - internal/bootstrap/install-rpms: + internal/bootstrap/install-rpms/sources/common: no-create-mountpoints: true description: | - Setup install-rpm scripts and collaterals for bootstrapping + install-rpms script and common rpms list units: - - build: | - cp -a bootstrap/install-rpms /dest + - files: + - from: . + paths: + - 'bootstrap/install-rpms/install-rpms.bash' + - 'bootstrap/install-rpms/rpms-common' + + internal/bootstrap/install-rpms/sources/build: + no-create-mountpoints: true + description: | + build specific rpms list + units: + - files: + - from: . + paths: + - 'bootstrap/install-rpms/rpms-build' + + internal/bootstrap/install-rpms/sources/devel: + no-create-mountpoints: true + description: | + devel/test specific rpms list + units: + - files: + - from: . + paths: + - 'bootstrap/install-rpms/rpms-devel' internal/bootstrap/dnf-dirs: no-create-mountpoints: true @@ -112,33 +150,44 @@ images: mkdir -p /dest/var/lib/dnf chmod 755 /dest/var/lib/dnf - internal/bootstrap: + internal/bootstrap-build: description: | - Final bootstrap layer used as floor to build base images. + Final bootstrap layer used as floor for build images units: - image: .%internal/bootstrap/dnf-dirs - image: .%internal/bootstrap/extract - image: .%internal/bootstrap/eext-repos - - image: .%internal/bootstrap/install-rpms + - image: .%internal/bootstrap/install-rpms/sources/common + - image: .%internal/bootstrap/install-rpms/sources/build + + internal/bootstrap-devel: + description: | + Final bootstrap layer used as floor for devel images + units: + - image: .%internal/bootstrap/dnf-dirs + - image: .%internal/bootstrap/extract + - image: .%internal/bootstrap/eext-repos + - image: .%internal/bootstrap/install-rpms/sources/common + - image: .%internal/bootstrap/install-rpms/sources/devel base-image-build: units: - - floor: .%internal/bootstrap + - floor: .%internal/bootstrap-build sources: [] build: | echo "install rpms" - /install-rpms/install-rpms.bash \ - --common-rpms-file /install-rpms/rpms-common \ - --extra-rpms-file /install-rpms/rpms-build + bash /bootstrap/install-rpms/install-rpms.bash \ + --common-rpms-file /bootstrap/install-rpms/rpms-common \ + --extra-rpms-file /bootstrap/install-rpms/rpms-build base-image-devel: units: - - floor: .%internal/bootstrap + - floor: .%internal/bootstrap-devel sources: [] build: | - /install-rpms/install-rpms.bash \ - --common-rpms-file /install-rpms/rpms-common \ - --extra-rpms-file /install-rpms/rpms-devel + bash /bootstrap/install-rpms/install-rpms.bash \ + --common-rpms-file /bootstrap/install-rpms/rpms-common \ + --extra-rpms-file /bootstrap/install-rpms/rpms-devel go-binaries: description: | @@ -148,27 +197,51 @@ images: units: - mappings: /src/static: .%go/static + sources: [] build: | mkdir -p /dest/usr cp -a /src/static/usr/bin /dest/usr/bin chmod 555 /dest/usr/bin - eext-configfiles: + src/configfiles: + description: | + Configuration files for eext units: - - build: | + - files: + - from: . + match: + - 'configfiles/*' + + src/pki: + description: | + pki files for eext + units: + - files: + - from: . + match: + - 'pki/**/*' + - 'pki/**/**/*' + + eext-staticfiles: + no-create-mountpoints: true + units: + - mappings: + /src/1: .%src/configfiles + /src/2: .%src/pki + sources: [] + build: | mkdir -p $DESTDIR/usr/share/eext - cp -a ./configfiles/* $DESTDIR/usr/share/eext + cp /src/1/configfiles/* $DESTDIR/usr/share/eext mkdir -p $DESTDIR/etc/pki/eext - cp -a ./pki/* $DESTDIR/etc/pki/eext - mkdir -p $DESTDIR/etc - touch $DESTDIR/etc/resolv.conf + cp -a /src/2/pki/* $DESTDIR/etc/pki/eext eext: units: - image: .%base-image-build - image: .%go-binaries - - image: .%eext-configfiles - - build: | + - image: .%eext-staticfiles + - sources: [] + build: | mkdir -p $DESTDIR/etc touch $DESTDIR/etc/resolv.conf @@ -193,12 +266,12 @@ images: - build: | mkdir -p /dest/var/cache/go mkdir -p /dest/var/ext - - floor: .%internal/bootstrap + - floor: .%internal/bootstrap-devel sources: [] build: | - /install-rpms/install-rpms.bash \ - --common-rpms-file /install-rpms/rpms-common \ - --extra-rpms-file /install-rpms/rpms-devel + bash /bootstrap/install-rpms/install-rpms.bash \ + --common-rpms-file /bootstrap/install-rpms/rpms-common \ + --extra-rpms-file /bootstrap/install-rpms/rpms-devel touch $DESTDIR/etc/resolv.conf entry: diff --git a/bootstrap/eext-repos/generate.bash b/bootstrap/eext-repos/generate.bash index 6c9d1c5..4dc936a 100755 --- a/bootstrap/eext-repos/generate.bash +++ b/bootstrap/eext-repos/generate.bash @@ -6,8 +6,9 @@ set -x generate_repo_files() { WORKING_DIR=`pwd` pushd ${COLLATERALS_DIR} - ./generate-repo-file.bash ./eext-repos-build.repo.template ./repos-build.env "${WORKING_DIR}/eext-repos-build.repo" - ./generate-repo-file.bash ./eext-repos-devel.repo.template ./repos-devel.env "${WORKING_DIR}/eext-repos-devel.repo" + EEXT_REPOS_DIR="/bootstrap/eext-repos" + bash ./generate-repo-file.bash "${EEXT_REPOS_DIR}/eext-repos-build.repo.template" "${EEXT_REPOS_DIR}/repos-build.env" "${WORKING_DIR}/eext-repos-build.repo" + bash ./generate-repo-file.bash "${EEXT_REPOS_DIR}/eext-repos-devel.repo.template" "${EEXT_REPOS_DIR}/repos-devel.env" "${WORKING_DIR}/eext-repos-devel.repo" popd mkdir -p /dest/etc/yum.repos.d chmod 755 /dest/etc/yum.repos.d @@ -20,8 +21,15 @@ generate_rpm() { pushd rpmbuild mkdir SOURCES SPECS - cp "${COLLATERALS_DIR}"/eext-repos.spec SPECS/ - cp "${WORKING_DIR}"/*.repo "${COLLATERALS_DIR}"/*.pem SOURCES/ + # Copy the spec file + cp "${COLLATERALS_DIR}"/eext-repos.spec SPECS + # Copy the pem files and generated repos file to SOURCES + for pemFile in "${PUBKEY_PEM_FILES[@]}" + do + cp "$pemFile" SOURCES/ + done + cp "${WORKING_DIR}"/*.repo SOURCES/ + rpmbuild --define "_topdir `pwd`" \ --define "eext_alma_version ${DNF_DISTRO_REPO_VERSION}" \ --define "eext_alma_release ${DNF_DISTRO_REPO_RELEASE}" \ @@ -30,6 +38,7 @@ generate_rpm() { --define "clamp_mtime_to_source_date_epoch 1" \ --define "_buildhost eext-buildhost" \ --define "_build_name_fmt %%{NAME}.rpm" \ + --define "__os_install_post /bin/true" \ -ba ./SPECS/eext-repos.spec if [ ! -f "./RPMS/eext-repos-build.rpm" ]; then @@ -49,15 +58,19 @@ generate_rpm() { setup_gpg_keys() { mkdir -p /dest/usr/share/eext-gpg-keys chmod 755 /dest/usr/share/eext-gpg-keys - cp "${COLLATERALS_DIR}"/*.pem /dest/usr/share/eext-gpg-keys/ + # Copy the pem files to the generated image + for pemFile in "${PUBKEY_PEM_FILES[@]}" + do + cp "$pemFile" /dest/usr/share/eext-gpg-keys/ + done } usage() { - echo "Usage: $0 " + echo "Usage: $0 i [ pemfile1.pem ... ]" exit 1 } -if [ $# -ne 1 ]; then +if [ $# -lt 1 ]; then usage fi @@ -66,13 +79,28 @@ if [ ! -d "$COLLATERALS_DIR" ]; then echo "Error: Collaterals directory '$COLLATERALS_DIR' not found." exit 1 fi +shift -TEMPLATE_FILE=$1 set -a source "$COLLATERALS_DIR/repos-common.env" set +a export ARCH=$(uname -m) + +PUBKEY_PEM_FILES=() +for arg in "$@"; do + if [[ "$arg" != *.pem ]]; then + echo "Error: '$arg' is not a .pem file." + exit 1 + fi + + if [ ! -f "$arg" ]; then + echo "Error: File '$arg' not found." + exit 1 + fi + PUBKEY_PEM_FILES+=("$arg") +done + generate_repo_files generate_rpm setup_gpg_keys diff --git a/bootstrap/extract/extract.bash b/bootstrap/extract/extract.bash index 75f5014..b41bc13 100755 --- a/bootstrap/extract/extract.bash +++ b/bootstrap/extract/extract.bash @@ -12,9 +12,9 @@ if [ $# -ne 1 ]; then usage fi -COLLATERALS_DIR=$1 -if [ ! -d "$COLLATERALS_DIR" ]; then - echo "Error: Collaterals directory '$COLLATERALS_DIR' not found." +CHECKSUM_FILE=$1 +if [ ! -f "$CHECKSUM_FILE" ]; then + echo "Error: Checksum file '$CHECKSUM_FILE' not found." exit 1 fi @@ -32,7 +32,7 @@ bootstrap_url="${DNF_HOST}/${bootstrap_file_repodir_path}/${bootstrap_filename}" wget ${bootstrap_url} # Validate downloaded tarball -grep "${bootstrap_filename}" "${COLLATERALS_DIR}/CHECKSUM" | sha256sum -wc +grep "${bootstrap_filename}" "${CHECKSUM_FILE}" | sha256sum -wc # Extract tarball and setup rootfs # This is a nested tarball, the real rootfs is in layer.tar diff --git a/bootstrap/install-rpms/rpms-test b/bootstrap/install-rpms/rpms-test deleted file mode 100644 index f82c801..0000000 --- a/bootstrap/install-rpms/rpms-test +++ /dev/null @@ -1,4 +0,0 @@ -# Specify extra rpms to be installed for the testfloor -# Use a path specification for local rpm, otherwise it'll be installed -# from one of the repos used for bootstrapping. -golang From c9aa0e57a478f8683e2bff45d5c6417c8c931b58 Mon Sep 17 00:00:00 2001 From: Arun Ajith S Date: Wed, 11 Sep 2024 12:06:50 +0000 Subject: [PATCH 3/5] Avoid using .%go/static The go generator would pull in the current repo's commit hash into the snapshot hash. Instead, we'll running go build ourselves on a floor with the go toolchain and modules already present. We also map in the subset of sources needed to build the eext static binary. .%go/modules would have sufficed if not for the fact that it also pulls in all sources to do go mod tidy. So we just enable the network and forego any benefit from the go generator module cache. --- barney.yaml | 57 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/barney.yaml b/barney.yaml index 9ef4063..15fcc94 100644 --- a/barney.yaml +++ b/barney.yaml @@ -189,19 +189,58 @@ images: --common-rpms-file /bootstrap/install-rpms/rpms-common \ --extra-rpms-file /bootstrap/install-rpms/rpms-devel + go-buildfloor: + description: | + Build floor for the eext go binary. + This buildfloor is used to manually run go build instead of using + the go generator to avoid adding the repo commit hash into the + snapshot hash. + + TODO: We cannot use .%go/modules here as one of the image units + because that also maps in all the sources to run go mod tidy. + As a workaround, we're turning on the network and foregoing any + benefits from the go generator's cache. + units: + - image: barney.ci/golang%dev + - image: barney.ci/alpine%network + entry: + mutables: + - /go/pkg/mod + + go-sources: + description: | + Image with source files need to build the eext static binary. + units: + - files: + - from: . + match: + - 'go.mod' + - 'go.sum' + - 'main.go' + - 'cmd/*.go' + - 'dnfconfig/*.go' + - 'impl/*.go' + - 'manifest/*.go' + - 'srcconfig/*.go' + - 'util/*.go' + go-binaries: description: | - This image is a copy of .%go/static, but with /usr/bin - permissions changed to 0555, so that it can combine properly - with redhat-style images. + We avoid using the go generator directly for two reasons: + 1. We want redhat style directory permissions of 0555 + 2. We want to present a filtered view of sources, mapping in only + the go sources. This makes sure we use a content hash of the go sources + and not the SHA of the repo to generate the snapshot hash of .%go-binaries. units: - - mappings: - /src/static: .%go/static - sources: [] + - sources: [] + mappings: + /src: .%go-sources + floor: .%go-buildfloor build: | - mkdir -p /dest/usr - cp -a /src/static/usr/bin /dest/usr/bin - chmod 555 /dest/usr/bin + mkdir -p /dest/usr/bin + chmod 0555 /dest/usr/bin + cd /src + CGO_ENABLED=0 go build -o /dest/usr/bin -ldflags '-extldflags "-static"' ./... src/configfiles: description: | From 0e9d27ae05fc173b43930f94f70668bb775abd5f Mon Sep 17 00:00:00 2001 From: Arun Ajith S Date: Mon, 23 Sep 2024 14:33:15 +0000 Subject: [PATCH 4/5] Remove trailing whitespaces --- barney.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/barney.yaml b/barney.yaml index 15fcc94..197991e 100644 --- a/barney.yaml +++ b/barney.yaml @@ -12,7 +12,7 @@ images: paths: - 'bootstrap/extract/extract.bash' - 'bootstrap/extract/CHECKSUM' - + internal/bootstrap/extract/floor: entry: mutables: @@ -64,7 +64,7 @@ images: - 'bootstrap/eext-repos/*' - 'pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem' - 'pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem' - + internal/bootstrap/eext-repos/floor: description: | Alpine floor with the eext-repos src directory mapped in. @@ -205,7 +205,7 @@ images: - image: barney.ci/alpine%network entry: mutables: - - /go/pkg/mod + - /go/pkg/mod go-sources: description: | @@ -238,7 +238,7 @@ images: floor: .%go-buildfloor build: | mkdir -p /dest/usr/bin - chmod 0555 /dest/usr/bin + chmod 0555 /dest/usr/bin cd /src CGO_ENABLED=0 go build -o /dest/usr/bin -ldflags '-extldflags "-static"' ./... @@ -272,7 +272,7 @@ images: mkdir -p $DESTDIR/usr/share/eext cp /src/1/configfiles/* $DESTDIR/usr/share/eext mkdir -p $DESTDIR/etc/pki/eext - cp -a /src/2/pki/* $DESTDIR/etc/pki/eext + cp -a /src/2/pki/* $DESTDIR/etc/pki/eext eext: units: From a0e72a80ec3fc5090cd22f5cbd824a5bfc10883a Mon Sep 17 00:00:00 2001 From: Arun Ajith S Date: Mon, 23 Sep 2024 14:41:09 +0000 Subject: [PATCH 5/5] Fix yamllint 'wrong indentation' errors --- barney.yaml | 74 ++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/barney.yaml b/barney.yaml index 197991e..33673e5 100644 --- a/barney.yaml +++ b/barney.yaml @@ -8,10 +8,10 @@ images: internal/bootstrap/extract/sources: units: - files: - - from: . - paths: - - 'bootstrap/extract/extract.bash' - - 'bootstrap/extract/CHECKSUM' + - from: . + paths: + - 'bootstrap/extract/extract.bash' + - 'bootstrap/extract/CHECKSUM' internal/bootstrap/extract/floor: entry: @@ -59,11 +59,11 @@ images: internal/bootstrap/eext-repos/sources: units: - files: - - from: . - match: - - 'bootstrap/eext-repos/*' - - 'pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem' - - 'pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem' + - from: . + match: + - 'bootstrap/eext-repos/*' + - 'pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem' + - 'pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem' internal/bootstrap/eext-repos/floor: description: | @@ -115,10 +115,10 @@ images: install-rpms script and common rpms list units: - files: - - from: . - paths: - - 'bootstrap/install-rpms/install-rpms.bash' - - 'bootstrap/install-rpms/rpms-common' + - from: . + paths: + - 'bootstrap/install-rpms/install-rpms.bash' + - 'bootstrap/install-rpms/rpms-common' internal/bootstrap/install-rpms/sources/build: no-create-mountpoints: true @@ -126,9 +126,9 @@ images: build specific rpms list units: - files: - - from: . - paths: - - 'bootstrap/install-rpms/rpms-build' + - from: . + paths: + - 'bootstrap/install-rpms/rpms-build' internal/bootstrap/install-rpms/sources/devel: no-create-mountpoints: true @@ -136,9 +136,9 @@ images: devel/test specific rpms list units: - files: - - from: . - paths: - - 'bootstrap/install-rpms/rpms-devel' + - from: . + paths: + - 'bootstrap/install-rpms/rpms-devel' internal/bootstrap/dnf-dirs: no-create-mountpoints: true @@ -212,17 +212,17 @@ images: Image with source files need to build the eext static binary. units: - files: - - from: . - match: - - 'go.mod' - - 'go.sum' - - 'main.go' - - 'cmd/*.go' - - 'dnfconfig/*.go' - - 'impl/*.go' - - 'manifest/*.go' - - 'srcconfig/*.go' - - 'util/*.go' + - from: . + match: + - 'go.mod' + - 'go.sum' + - 'main.go' + - 'cmd/*.go' + - 'dnfconfig/*.go' + - 'impl/*.go' + - 'manifest/*.go' + - 'srcconfig/*.go' + - 'util/*.go' go-binaries: description: | @@ -247,19 +247,19 @@ images: Configuration files for eext units: - files: - - from: . - match: - - 'configfiles/*' + - from: . + match: + - 'configfiles/*' src/pki: description: | pki files for eext units: - files: - - from: . - match: - - 'pki/**/*' - - 'pki/**/**/*' + - from: . + match: + - 'pki/**/*' + - 'pki/**/**/*' eext-staticfiles: no-create-mountpoints: true