Refactor the code to be more cache aware

Using Snaipe's new bfg feature that allows file inlining to make the bootstrap, base image build and eext builds more bsy cache aware. Roll-With: barney.ci/barneyfile#c5b8739b74ad067be0942533c286b735803d8b72
aristanetworks · Sep 11, 2024 · a056dcf · a056dcf
1 parent 4b043b6
commit a056dcf
Show file tree

Hide file tree

Showing 4 changed files with 153 additions and 56 deletions.
diff --git a/barney.yaml b/barney.yaml
@@ -5,7 +5,15 @@ generators:
 
 images:
 
-  internal/bootstrap/extract-floor:
+  internal/bootstrap/extract/sources:
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/extract/extract.bash'
+            - 'bootstrap/extract/CHECKSUM'
+
+  internal/bootstrap/extract/floor:
     entry:
       mutables:
         - /var/work
@@ -15,9 +23,7 @@ images:
       - image: barney.ci/alpine%pkg/wget
       - image: barney.ci/alpine%network
       - image: barney.ci/alpine%apk-finalizers
-      - build: |
-          mkdir -p /dest/var/work
-          cp -a bootstrap/extract /dest
+      - image: .%internal/bootstrap/extract/sources
 
   internal/bootstrap/extract:
     no-create-mountpoints: true
@@ -40,17 +46,26 @@ images:
         - /var/cache
         - /var/lib/dnf
     units:
-      - floor: .%internal/bootstrap/extract-floor
+      - floor: .%internal/bootstrap/extract/floor
         entry:
           env:
             DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory}
         sources: []
         build: |
           cd /var/work
-          /extract/extract.bash /extract
+          bash /bootstrap/extract/extract.bash /bootstrap/extract/CHECKSUM
           touch $DESTDIR/etc/resolv.conf
 
-  internal/bootstrap/eext-repos-floor:
+  internal/bootstrap/eext-repos/sources:
+    units:
+      - files:
+        - from: .
+          match:
+            - 'bootstrap/eext-repos/*'
+            - 'pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem'
+            - 'pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem'
+
+  internal/bootstrap/eext-repos/floor:
     description: |
       Alpine floor with the eext-repos src directory mapped in.
       gettext provides envsubst used by generate-repo-file.bash called from generate.bash
@@ -65,11 +80,10 @@ images:
       - image: barney.ci/alpine%pkg/gettext
       - image: barney.ci/alpine%pkg/rpm-dev
       - image: barney.ci/alpine%apk-finalizers
-      - build: |
+      - image: .%internal/bootstrap/eext-repos/sources
+      - sources: []
+        build: |
           mkdir -p /dest/var/work
-          cp -a bootstrap/eext-repos /dest
-          cp -a pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /dest/eext-repos
-          cp -a pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem /dest/eext-repos
 
   internal/bootstrap/eext-repos:
     no-create-mountpoints: true
@@ -85,22 +99,46 @@ images:
       The RPMs are named eext-repos-build and eext-repos-devel with approiate architecture
       and .rpm extensions. They are installed in /RPMS.
     units:
-      - floor: .%internal/bootstrap/eext-repos-floor
+      - floor: .%internal/bootstrap/eext-repos/floor
         entry:
           env:
             DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory}
         sources: []
         build: |
           cd /var/work
-          /eext-repos/generate.bash /eext-repos
+          # First argument is collaterals dir, further args are pem files
+          bash /bootstrap/eext-repos/generate.bash /bootstrap/eext-repos /pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem
 
-  internal/bootstrap/install-rpms:
+  internal/bootstrap/install-rpms/sources/common:
     no-create-mountpoints: true
     description: |
-      Setup install-rpm scripts and collaterals for bootstrapping
+      install-rpms script and common rpms list
     units:
-      - build: |
-          cp -a bootstrap/install-rpms /dest
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/install-rpms/install-rpms.bash'
+            - 'bootstrap/install-rpms/rpms-common'
+
+  internal/bootstrap/install-rpms/sources/build:
+    no-create-mountpoints: true
+    description: |
+      build specific rpms list
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/install-rpms/rpms-build'
+
+  internal/bootstrap/install-rpms/sources/devel:
+    no-create-mountpoints: true
+    description: |
+      devel/test specific rpms list
+    units:
+      - files:
+        - from: .
+          paths:
+            - 'bootstrap/install-rpms/rpms-devel'
 
   internal/bootstrap/dnf-dirs:
     no-create-mountpoints: true
@@ -112,33 +150,44 @@ images:
           mkdir -p /dest/var/lib/dnf
           chmod 755 /dest/var/lib/dnf
 
-  internal/bootstrap:
+  internal/bootstrap-build:
     description: |
-      Final bootstrap layer used as floor to build base images.
+      Final bootstrap layer used as floor for build images
     units:
       - image: .%internal/bootstrap/dnf-dirs
       - image: .%internal/bootstrap/extract
       - image: .%internal/bootstrap/eext-repos
-      - image: .%internal/bootstrap/install-rpms
+      - image: .%internal/bootstrap/install-rpms/sources/common
+      - image: .%internal/bootstrap/install-rpms/sources/build
+
+  internal/bootstrap-devel:
+    description: |
+      Final bootstrap layer used as floor for devel images
+    units:
+      - image: .%internal/bootstrap/dnf-dirs
+      - image: .%internal/bootstrap/extract
+      - image: .%internal/bootstrap/eext-repos
+      - image: .%internal/bootstrap/install-rpms/sources/common
+      - image: .%internal/bootstrap/install-rpms/sources/devel
 
   base-image-build:
     units:
-      - floor: .%internal/bootstrap
+      - floor: .%internal/bootstrap-build
         sources: []
         build: |
           echo "install rpms"
-          /install-rpms/install-rpms.bash \
-            --common-rpms-file /install-rpms/rpms-common \
-            --extra-rpms-file /install-rpms/rpms-build
+          bash /bootstrap/install-rpms/install-rpms.bash \
+            --common-rpms-file /bootstrap/install-rpms/rpms-common \
+            --extra-rpms-file /bootstrap/install-rpms/rpms-build
 
   base-image-devel:
     units:
-      - floor: .%internal/bootstrap
+      - floor: .%internal/bootstrap-devel
         sources: []
         build: |
-          /install-rpms/install-rpms.bash \
-            --common-rpms-file /install-rpms/rpms-common \
-            --extra-rpms-file /install-rpms/rpms-devel
+          bash /bootstrap/install-rpms/install-rpms.bash \
+            --common-rpms-file /bootstrap/install-rpms/rpms-common \
+            --extra-rpms-file /bootstrap/install-rpms/rpms-devel
 
   go-binaries:
     description: |
@@ -148,27 +197,51 @@ images:
     units:
       - mappings:
           /src/static: .%go/static
+        sources: []
         build: |
           mkdir -p /dest/usr
           cp -a /src/static/usr/bin /dest/usr/bin
           chmod 555 /dest/usr/bin
 
-  eext-configfiles:
+  src/configfiles:
+    description: |
+      Configuration files for eext
     units:
-      - build: |
+      - files:
+        - from: .
+          match:
+            - 'configfiles/*'
+
+  src/pki:
+    description: |
+      pki files for eext
+    units:
+      - files:
+        - from: .
+          match:
+            - 'pki/**/*'
+            - 'pki/**/**/*'
+
+  eext-staticfiles:
+    no-create-mountpoints: true
+    units:
+      - mappings:
+          /src/1: .%src/configfiles
+          /src/2: .%src/pki
+        sources: []
+        build: |
           mkdir -p $DESTDIR/usr/share/eext
-          cp -a ./configfiles/* $DESTDIR/usr/share/eext
+          cp /src/1/configfiles/* $DESTDIR/usr/share/eext
           mkdir -p $DESTDIR/etc/pki/eext
-          cp -a ./pki/* $DESTDIR/etc/pki/eext
-          mkdir -p $DESTDIR/etc
-          touch $DESTDIR/etc/resolv.conf
+          cp -a /src/2/pki/* $DESTDIR/etc/pki/eext 
 
   eext:
     units:
       - image: .%base-image-build
       - image: .%go-binaries
-      - image: .%eext-configfiles
-      - build: |
+      - image: .%eext-staticfiles
+      - sources: []
+        build: |
           mkdir -p $DESTDIR/etc
           touch $DESTDIR/etc/resolv.conf
 
@@ -193,12 +266,12 @@ images:
       - build: |
           mkdir -p /dest/var/cache/go
           mkdir -p /dest/var/ext
-      - floor: .%internal/bootstrap
+      - floor: .%internal/bootstrap-devel
         sources: []
         build: |
-          /install-rpms/install-rpms.bash \
-            --common-rpms-file /install-rpms/rpms-common \
-            --extra-rpms-file /install-rpms/rpms-devel
+          bash /bootstrap/install-rpms/install-rpms.bash \
+            --common-rpms-file /bootstrap/install-rpms/rpms-common \
+            --extra-rpms-file /bootstrap/install-rpms/rpms-devel
           touch $DESTDIR/etc/resolv.conf
 
     entry:

diff --git a/bootstrap/eext-repos/generate.bash b/bootstrap/eext-repos/generate.bash
@@ -6,8 +6,9 @@ set -x
 generate_repo_files() {
    WORKING_DIR=`pwd`
    pushd ${COLLATERALS_DIR}
-   ./generate-repo-file.bash ./eext-repos-build.repo.template ./repos-build.env "${WORKING_DIR}/eext-repos-build.repo"
-   ./generate-repo-file.bash ./eext-repos-devel.repo.template ./repos-devel.env "${WORKING_DIR}/eext-repos-devel.repo"
+   EEXT_REPOS_DIR="/bootstrap/eext-repos"
+   bash ./generate-repo-file.bash "${EEXT_REPOS_DIR}/eext-repos-build.repo.template" "${EEXT_REPOS_DIR}/repos-build.env" "${WORKING_DIR}/eext-repos-build.repo"
+   bash ./generate-repo-file.bash "${EEXT_REPOS_DIR}/eext-repos-devel.repo.template" "${EEXT_REPOS_DIR}/repos-devel.env" "${WORKING_DIR}/eext-repos-devel.repo"
    popd
    mkdir -p /dest/etc/yum.repos.d
    chmod 755 /dest/etc/yum.repos.d
@@ -20,8 +21,15 @@ generate_rpm() {
    pushd rpmbuild
    mkdir SOURCES SPECS
 
-   cp "${COLLATERALS_DIR}"/eext-repos.spec SPECS/
-   cp "${WORKING_DIR}"/*.repo "${COLLATERALS_DIR}"/*.pem SOURCES/
+   # Copy the spec file
+   cp "${COLLATERALS_DIR}"/eext-repos.spec SPECS
+   # Copy the pem files and generated repos file to SOURCES
+   for pemFile in "${PUBKEY_PEM_FILES[@]}"
+   do
+     cp "$pemFile" SOURCES/
+   done
+   cp "${WORKING_DIR}"/*.repo SOURCES/
+
    rpmbuild --define "_topdir `pwd`" \
             --define "eext_alma_version ${DNF_DISTRO_REPO_VERSION}" \
             --define "eext_alma_release ${DNF_DISTRO_REPO_RELEASE}"  \
@@ -30,6 +38,7 @@ generate_rpm() {
             --define "clamp_mtime_to_source_date_epoch 1" \
             --define "_buildhost eext-buildhost" \
             --define "_build_name_fmt %%{NAME}.rpm" \
+            --define "__os_install_post /bin/true" \
             -ba ./SPECS/eext-repos.spec
 
    if [ ! -f "./RPMS/eext-repos-build.rpm" ]; then
@@ -49,15 +58,19 @@ generate_rpm() {
 setup_gpg_keys() {
    mkdir -p /dest/usr/share/eext-gpg-keys
    chmod 755  /dest/usr/share/eext-gpg-keys
-   cp "${COLLATERALS_DIR}"/*.pem /dest/usr/share/eext-gpg-keys/
+   # Copy the pem files to the generated image
+   for pemFile in "${PUBKEY_PEM_FILES[@]}"
+   do
+     cp "$pemFile" /dest/usr/share/eext-gpg-keys/
+   done
 }
 
 usage() {
-   echo "Usage: $0 <collaterals_dir>"
+   echo "Usage: $0 <collaterals_dir>i [ pemfile1.pem ... ]"
    exit 1
 }
 
-if [ $# -ne 1 ]; then
+if [ $# -lt 1 ]; then
    usage
 fi
 
@@ -66,13 +79,28 @@ if [ ! -d "$COLLATERALS_DIR" ]; then
    echo "Error: Collaterals directory '$COLLATERALS_DIR' not found."
    exit 1
 fi
+shift
 
-TEMPLATE_FILE=$1
 set -a
 source "$COLLATERALS_DIR/repos-common.env"
 set +a
 export ARCH=$(uname -m)
 
+
+PUBKEY_PEM_FILES=()
+for arg in "$@"; do
+   if [[ "$arg" != *.pem ]]; then
+      echo "Error: '$arg' is not a .pem file."
+      exit 1
+   fi
+
+   if [ ! -f "$arg" ]; then
+      echo "Error: File '$arg' not found."
+      exit 1
+   fi
+   PUBKEY_PEM_FILES+=("$arg")
+done
+
 generate_repo_files
 generate_rpm
 setup_gpg_keys
diff --git a/bootstrap/extract/extract.bash b/bootstrap/extract/extract.bash
@@ -12,9 +12,9 @@ if [ $# -ne 1 ]; then
    usage
 fi
 
-COLLATERALS_DIR=$1
-if [ ! -d "$COLLATERALS_DIR" ]; then
-   echo "Error: Collaterals directory '$COLLATERALS_DIR' not found."
+CHECKSUM_FILE=$1
+if [ ! -f "$CHECKSUM_FILE" ]; then
+   echo "Error: Checksum file '$CHECKSUM_FILE' not found."
    exit 1
 fi
 
@@ -32,7 +32,7 @@ bootstrap_url="${DNF_HOST}/${bootstrap_file_repodir_path}/${bootstrap_filename}"
 wget ${bootstrap_url}
 
 # Validate downloaded tarball
-grep "${bootstrap_filename}" "${COLLATERALS_DIR}/CHECKSUM" | sha256sum -wc
+grep "${bootstrap_filename}" "${CHECKSUM_FILE}" | sha256sum -wc
 
 # Extract tarball and setup rootfs
 # This is a nested tarball, the real rootfs is in layer.tar

diff --git a/bootstrap/install-rpms/rpms-test b/bootstrap/install-rpms/rpms-test