Barney image changes for devel workflow

barney.yaml

@@ -5,29 +5,28 @@
 
 images:
 
-  internal/bootstrap/iso-extraction-floor:
+  internal/bootstrap/extract/sources:
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/extract/extract.bash'
+            - 'bootstrap/extract/CHECKSUM'
+
+  internal/bootstrap/extract/floor:
     entry:
       mutables:
         - /var/work
     units:
       - image: barney.ci/alpine%pkg/alpine-base
-      - image: barney.ci/alpine%pkg/gettext
+      - image: barney.ci/alpine%pkg/bash
       - image: barney.ci/alpine%pkg/wget
       - image: barney.ci/alpine%network
       - image: barney.ci/alpine%apk-finalizers
-      - build: |
-          mkdir -p /dest/var/work
+      - image: .%internal/bootstrap/extract/sources
 
-  internal/bootstrap/src-deps:
+  internal/bootstrap/extract:
     no-create-mountpoints: true
-    units:
-      - build: |
-          cp -a bootstrap/* /dest/
-          cp -a pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /dest/
-          cp -a pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem /dest/
-          rm -f /dest/README*
-
-  internal/bootstrap:
     description: |
       Minimal bootstrapping environment used to bootstrap almalinux.
 
@@ -36,11 +35,7 @@
       is readily available on the mirror. This is seldom changed and
       any change here will update all eext snapshots.
 
-      We wipe out the dnf/yum repo configuration in the tarball,
-      and setup our own curated set of frozen almalinux repos.
-
-      We also include an install-rpms script which is used to bootstrap
-      the almalinux image.
+      We wipe out the dnf/yum repo configuration in the tarball.
     entry:
       share-net: true
       mounts:
@@ -51,90 +46,244 @@
         - /var/cache
         - /var/lib/dnf
     units:
-      - floor: .%internal/bootstrap/iso-extraction-floor
+      - floor: .%internal/bootstrap/extract/floor
         entry:
           env:
             DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory}
-            BOOTSTRAP_PATH: eext-sources/bootstrap/CentOS-Stream
-            BOOTSTRAP_FILE: CentOS-Stream-Container-Base-9-20240715.0
-            BOOTSTRAP_EXTENSION: tar.xz
-            DNF_DISTRO_REPO: eext-alma-vault
-            DNF_DISTRO_REPO_VERSION: "9.3"
-            DNF_EPEL_REPO: eext-snapshots-local/epel9
-            DNF_EPEL_REPO_SNAPSHOT_VERSION: v20240127-1
-            DNF_EPEL_REPO_DISTRO_VERSION: "9"
         sources: []
-        mappings:
-          /src/bootstrap: .%internal/bootstrap/src-deps
         build: |
-          # URL of tarball with OS image
-          export ARCH=$(uname -m)
-          bootstrap_filename="${BOOTSTRAP_FILE}.${ARCH}.${BOOTSTRAP_EXTENSION}"
-          bootstrap_url="${DNF_HOST}/${BOOTSTRAP_PATH}/${bootstrap_filename}"
+          cd /var/work
+          bash /bootstrap/extract/extract.bash /bootstrap/extract/CHECKSUM
+          touch $DESTDIR/etc/resolv.conf
 
-          # Download the tarball into the mutable working dir
+  internal/bootstrap/eext-repos/sources:
+    units:
+      - files:
+        - from: .
+          match:
+            - 'bootstrap/eext-repos/*'
+            - 'pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem'
+            - 'pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem'
+
+  internal/bootstrap/eext-repos/floor:
+    description: |
+      Alpine floor with the eext-repos src directory mapped in.
+      gettext provides envsubst used by generate-repo-file.bash called from generate.bash
+      rpm-dev provides rpmbuild used by generate.bash.
+    entry:
+      mutables:
+        - /var/work
+        - /var/tmp
+    units:
+      - image: barney.ci/alpine%pkg/alpine-base
+      - image: barney.ci/alpine%pkg/bash
+      - image: barney.ci/alpine%pkg/gettext
+      - image: barney.ci/alpine%pkg/rpm-dev
+      - image: barney.ci/alpine%apk-finalizers
+      - image: .%internal/bootstrap/eext-repos/sources
+      - sources: []
+        build: |
+          mkdir -p /dest/var/work
+
+  internal/bootstrap/eext-repos:
+    no-create-mountpoints: true
+    description: |
+      Setup curated frozen dnf repo list for build reproducibility.
+      There're two files, eext-repo-build.repo and eext-repo-devel.repo. These files are
+      installed in /eext-repos directory.
+      We also install the eext-repo-build.repo in /etc/yum.repos.d to be used for base image
+      bootstrap.
+
+      We also build an rpm each with the contents being the repo file,
+      and instructions to install at /etc/yum.repos.d.
+      The RPMs are named eext-repos-build and eext-repos-devel with approiate architecture
+      and .rpm extensions. They are installed in /RPMS.
+    units:
+      - floor: .%internal/bootstrap/eext-repos/floor
+        entry:
+          env:
+            DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory}
+        sources: []
+        build: |
           cd /var/work
-          wget ${bootstrap_url}
-
-          # Validate downloaded tarball
-          grep "${bootstrap_filename}" /src/bootstrap/CHECKSUM | sha256sum -wc
-
-          # Extract tarball and setup rootfs
-          # This is a nested tarball, the real rootfs is in layer.tar
-          # Extract the firt level tarball inside the extr subdirectory
-          # within the working directory and and then extract the
-          # second level layer.tar directly to /dest
-          mkdir extr
-          tar --strip-components=1 -C ./extr -xf ./${bootstrap_filename}
-          tar -xf ./extr/layer.tar -C /dest
-
-          # Now modify the extracted file system to remove unwanted
-          # stuff and add extra stuff.
-
-          # Remove any pre-configured yum repos
-          rm /dest/etc/yum.repos.d/*
-          # Setup install-rpms shell script
-          mkdir -p /dest/usr/bin
-          chmod 555 /dest/usr/bin
-          cp -a /src/bootstrap/install-rpms /dest/usr/bin/
-          chmod 0755 /dest/usr/bin/install-rpms
-          # Setup gpg keys
-          mkdir -p /dest/usr/share/distribution-gpg-keys
-          chmod 555 /dest/usr/share/distribution-gpg-keys
-          cp -a /src/bootstrap/*.pem /dest/usr/share/distribution-gpg-keys/
-          # Setup curated frozen dnf repo list for build reproducibility
-          envsubst < /src/bootstrap/eext-repos-build.repo.template > /dest/etc/yum.repos.d/eext-repos-build.repo
-
-  base-image:
-    units:
-      - floor: .%internal/bootstrap
+          # First argument is collaterals dir, further args are pem files
+          bash /bootstrap/eext-repos/generate.bash /bootstrap/eext-repos /pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem
+
+  internal/bootstrap/install-rpms/sources/common:
+    no-create-mountpoints: true
+    description: |
+      install-rpms script and common rpms list
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/install-rpms/install-rpms.bash'
+            - 'bootstrap/install-rpms/rpms-common'
+
+  internal/bootstrap/install-rpms/sources/build:
+    no-create-mountpoints: true
+    description: |
+      build specific rpms list
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/install-rpms/rpms-build'
+
+  internal/bootstrap/install-rpms/sources/devel:
+    no-create-mountpoints: true
+    description: |
+      devel/test specific rpms list
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/install-rpms/rpms-devel'
+
+  internal/bootstrap/dnf-dirs:
+    no-create-mountpoints: true
+    description: |
+      Setup empty directories needed by dnf
+    units:
+      - sources: []
+        build: |
+          mkdir -p /dest/var/lib/dnf
+          chmod 755 /dest/var/lib/dnf
+
+  internal/bootstrap-build:
+    description: |
+      Final bootstrap layer used as floor for build images
+    units:
+      - image: .%internal/bootstrap/dnf-dirs
+      - image: .%internal/bootstrap/extract
+      - image: .%internal/bootstrap/eext-repos
+      - image: .%internal/bootstrap/install-rpms/sources/common
+      - image: .%internal/bootstrap/install-rpms/sources/build
+
+  internal/bootstrap-devel:
+    description: |
+      Final bootstrap layer used as floor for devel images
+    units:
+      - image: .%internal/bootstrap/dnf-dirs
+      - image: .%internal/bootstrap/extract
+      - image: .%internal/bootstrap/eext-repos
+      - image: .%internal/bootstrap/install-rpms/sources/common
+      - image: .%internal/bootstrap/install-rpms/sources/devel
+
+  base-image-build:
+    units:
+      - floor: .%internal/bootstrap-build
         sources: []
-        build: install-rpms autoconf automake coreutils git rpm rpmdevtools rpm-build make mock python3-devel quilt
+        build: |
+          echo "install rpms"
+          bash /bootstrap/install-rpms/install-rpms.bash \
+            --common-rpms-file /bootstrap/install-rpms/rpms-common \
+            --extra-rpms-file /bootstrap/install-rpms/rpms-build
+
+  base-image-devel:
+    units:
+      - floor: .%internal/bootstrap-devel
+        sources: []
+        build: |
+          bash /bootstrap/install-rpms/install-rpms.bash \
+            --common-rpms-file /bootstrap/install-rpms/rpms-common \
+            --extra-rpms-file /bootstrap/install-rpms/rpms-devel
+
+  go-buildfloor:
+    description: |
+      Build floor for the eext go binary.
+      This buildfloor is used to manually run go build instead of using
+      the go generator to avoid adding the repo commit hash into the
+      snapshot hash.
+
+      TODO: We cannot use .%go/modules here as one of the image units
+      because that also maps in all the sources to run go mod tidy.
+      As a workaround, we're turning on the network and foregoing any
+      benefits from the go generator's cache.
+    units:
+      - image: barney.ci/golang%dev
+      - image: barney.ci/alpine%network
+    entry:
+      mutables:
+        - /go/pkg/mod
+
+  go-sources:
+    description: |
+      Image with source files need to build the eext static binary.
+    units:
+      - files:
+        - from: .
+          match:
+            - 'go.mod'
+            - 'go.sum'
+            - 'main.go'
+            - 'cmd/*.go'
+            - 'dnfconfig/*.go'
+            - 'impl/*.go'
+            - 'manifest/*.go'
+            - 'srcconfig/*.go'
+            - 'util/*.go'
 
   go-binaries:
     description: |
-      This image is a copy of .%go/static, but with /usr/bin
-      permissions changed to 0555, so that it can combine properly
-      with redhat-style images.
+      We avoid using the go generator directly for two reasons:
+      1. We want redhat style directory permissions of 0555
+      2. We want to present a filtered view of sources, mapping in only
+         the go sources. This makes sure we use a content hash of the go sources
+         and not the SHA of the repo to generate the snapshot hash of .%go-binaries.
+    units:
+      - sources: []
+        mappings:
+          /src: .%go-sources
+        floor: .%go-buildfloor
+        build: |
+          mkdir -p /dest/usr/bin
+          chmod 0555 /dest/usr/bin
+          cd /src
+          CGO_ENABLED=0 go build -o /dest/usr/bin -ldflags '-extldflags "-static"' ./...
+
+  src/configfiles:
+    description: |
+      Configuration files for eext
+    units:
+      - files:
+        - from: .
+          match:
+            - 'configfiles/*'
+
+  src/pki:
+    description: |
+      pki files for eext
+    units:
+      - files:
+        - from: .
+          match:
+            - 'pki/**/*'
+            - 'pki/**/**/*'
+
+  eext-staticfiles:
+    no-create-mountpoints: true
     units:
       - mappings:
-          /src/static: .%go/static
+          /src/1: .%src/configfiles
+          /src/2: .%src/pki
+        sources: []
         build: |
-          mkdir -p /dest/usr
-          cp -a /src/static/usr/bin /dest/usr/bin
-          chmod 555 /dest/usr/bin
+          mkdir -p $DESTDIR/usr/share/eext
+          cp /src/1/configfiles/* $DESTDIR/usr/share/eext
+          mkdir -p $DESTDIR/etc/pki/eext
+          cp -a /src/2/pki/* $DESTDIR/etc/pki/eext
 
   eext:
     units:
-      - image: .%base-image
+      - image: .%base-image-build
       - image: .%go-binaries
-      - build: |
-          mkdir -p $DESTDIR/usr/share/eext
-          cp -a ./configfiles/* $DESTDIR/usr/share/eext
-          mkdir -p $DESTDIR/etc/pki/eext
-          cp -a ./pki/* $DESTDIR/etc/pki/eext
+      - image: .%eext-staticfiles
+      - sources: []
+        build: |
           mkdir -p $DESTDIR/etc
           touch $DESTDIR/etc/resolv.conf
+
     entry:
       mutables: &eext-mutables
         - /var/eext
@@ -156,11 +305,14 @@
       - build: |
           mkdir -p /dest/var/cache/go
           mkdir -p /dest/var/ext
-      - floor: .%internal/bootstrap
+      - floor: .%internal/bootstrap-devel
         sources: []
         build: |
-          install-rpms autoconf automake coreutils golang git rpm rpmdevtools rpm-build make mock python3-devel quilt
+          bash /bootstrap/install-rpms/install-rpms.bash \
+            --common-rpms-file /bootstrap/install-rpms/rpms-common \
+            --extra-rpms-file /bootstrap/install-rpms/rpms-devel
           touch $DESTDIR/etc/resolv.conf
+
     entry:
       env:
         GOCACHE: /tmp/gocache