From 07cbac3ca6fa61714f80b4372055cf719427d5cf Mon Sep 17 00:00:00 2001 From: m2 <69128853+m2Giles@users.noreply.github.com> Date: Tue, 14 May 2024 16:03:50 -0400 Subject: [PATCH 1/2] chore(ci): 24.04 builder (#1295) Co-authored-by: Jorge O. Castro --- .github/workflows/reusable-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index fa23fa41363..da9e59c29e9 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -24,7 +24,7 @@ concurrency: jobs: build_container: name: image - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 continue-on-error: false outputs: image_full: ${{ steps.generate-outputs.outputs.image }} @@ -97,7 +97,7 @@ jobs: registry: cgr.dev/chainguard - name: Maximize build space - uses: ublue-os/remove-unwanted-software@v6 + uses: ublue-os/remove-unwanted-software@v7 - name: Check just syntax uses: ublue-os/just-action@v1 From 6b38313476623d90a1f672f909f72d179a5a34aa Mon Sep 17 00:00:00 2001 From: m2 <69128853+m2Giles@users.noreply.github.com> Date: Wed, 15 May 2024 10:26:16 -0400 Subject: [PATCH 2/2] feat(dev): improve just commands (#1297) Co-authored-by: Jorge O. Castro --- .vscode/tasks.json | 108 ++++++++++++++++++++++++++++ Justfile | 13 ++-- scripts/build-image.sh | 5 +- scripts/build-iso-installer-main.sh | 40 +++++++++++ scripts/build-iso.sh | 7 +- scripts/cleanup-dir.sh | 6 +- scripts/cleanup-images.sh | 2 +- scripts/common-build-iso.sh | 26 +++---- scripts/get-defaults.sh | 7 +- scripts/list-images.sh | 2 +- scripts/run-booted-guest.sh | 9 ++- scripts/run-booted-home.sh | 5 +- scripts/run-image.sh | 7 +- scripts/run-iso.sh | 7 +- scripts/sudoif.sh | 14 ++++ 15 files changed, 222 insertions(+), 36 deletions(-) create mode 100644 .vscode/tasks.json create mode 100644 scripts/build-iso-installer-main.sh create mode 100644 scripts/sudoif.sh diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000000..72791eeb6d6 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,108 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Connect to VM", + "dependsOn": [ + "Run-ISO", + "Open Browser" + ], + "problemMatcher": [] + }, + { + "label": "Open Browser", + "command": "${input:openSimpleBrowser}", + "problemMatcher": [] + }, + { + "label": "Build Container", + "command": "just", + "args": [ + "build", + "${input:outputChoice}" + ], + "problemMatcher": [], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Build ISO", + "command": "just", + "args": [ + "build-iso", + "${input:outputChoice}" + ], + "problemMatcher": [] + }, + { + "label": "Build ISO (testing)", + "command": "just", + "args": [ + "build-iso-installer-main", + "${input:outputChoice}" + ], + "problemMatcher": [] + }, + { + "label": "Run-ISO", + "command": "just", + "args": [ + "run-iso", + "${input:outputChoice}" + ], + "problemMatcher": [], + "isBackground": true + }, + { + "label": "List Images", + "command": "just", + "args": [ + "list-images" + ], + "problemMatcher": [] + }, + { + "label": "Check justfiles", + "command": "just", + "args": [ + "just-check" + ], + "problemMatcher": [] + }, + { + "label": "Run Container", + "command": "just", + "args": [ + "run-container", + "${input:outputChoice}" + ], + "problemMatcher": [] + } + ], + "inputs": [ + { + "id": "openSimpleBrowser", + "type": "command", + "command": "simpleBrowser.show", + "args": [ + "http://localhost:8006" + ] + }, + { + "id": "outputChoice", + "type": "pickString", + "description": "Choose which container to build", + "default": "bluefin", + "options": [ + "bluefin", + "bluefin-dx", + "bluefin latest", + "bluefin-dx latest", + "aurora", + "aurora-dx" + ] + } + ] +} diff --git a/Justfile b/Justfile index 4c1e2d9c7b3..82643893914 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,5 @@ export project_root := `git rev-parse --show-toplevel` +export git_branch := ` git branch --show-current` export gts := "39" export latest := "40" @@ -21,9 +22,9 @@ just-check: #!/usr/bin/bash find "${project_root}" -type f -name "*.just" | while read -r file; do echo "Checking syntax: $file" - just --unstable --fmt --check -f $file || { exit 1; } + just --unstable --fmt --check -f $file done - just --unstable --fmt --check -f ${project_root}/Justfile || { exit 1; } + just --unstable --fmt --check -f ${project_root}/Justfile # Fix Just Syntax [private] @@ -31,9 +32,9 @@ just-fix: #!/usr/bin/bash find "${project_root}" -type f -name "*.just" | while read -r file; do echo "Checking syntax: $file" - just --unstable --fmt -f $file || { exit 1; } + just --unstable --fmt -f $file done - just --unstable --fmt -f ${project_root}/Justfile || { exit 1; } + just --unstable --fmt -f ${project_root}/Justfile # Build Image build image="" target="" version="": @@ -55,6 +56,10 @@ run-booted-home image="" target="" version="": build-iso image="" target="" version="": @{{ project_root }}/scripts/build-iso.sh {{ image }} {{ target }} {{ version }} +# Create ISO from local dev build image - use build-container-installer:main +build-iso-installer-main image="" target="" version="": + @{{ project_root }}/scripts/build-iso-intstaller-main.sh {{ image }} {{ target }} {{ version }} + # Run ISO from local dev build image run-iso image="" target="" version="": @{{ project_root }}/scripts/run-iso.sh {{ image }} {{ target }} {{ version }} diff --git a/scripts/build-image.sh b/scripts/build-image.sh index 5e74aa0db33..be6c21a5d20 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -2,6 +2,9 @@ set -eo pipefail if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) +fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) fi # Get Inputs @@ -27,6 +30,6 @@ $container_mgr build -f Containerfile \ --build-arg="IMAGE_NAME=${tag}" \ --build-arg="IMAGE_FLAVOR=main" \ --build-arg="IMAGE_VENDOR=localhost" \ - --tag localhost/"${tag}":"${version}" \ + --tag localhost/"${tag}:${version}-${git_branch}" \ --target "${target}" \ "${project_root}" diff --git a/scripts/build-iso-installer-main.sh b/scripts/build-iso-installer-main.sh new file mode 100644 index 00000000000..d96bea41f49 --- /dev/null +++ b/scripts/build-iso-installer-main.sh @@ -0,0 +1,40 @@ +#!/usr/bin/bash +#shellcheck disable=SC2154 + +if [[ -z ${project_root} ]]; then + project_root=$(git rev-parse --show-toplevel) +fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) +fi + +# Common Build ISO +# shellcheck disable=SC1091 +. "${project_root}/scripts/common-build-iso.sh" + +if [[ ${container_mgr} =~ "podman" ]]; then + api_socket=/run/podman/podman.sock +elif [[ ${container_mgr} =~ "docker" ]]; then + api_socket=/var/run/docker.sock +fi + +# Make ISO +${container_mgr} run --rm --privileged \ + --volume "${api_socket}":/var/run/docker.sock \ + --volume "${workspace}"/scripts/files/build-iso-makefile-patch:/build-container-installer/container/Makefile \ + --volume "${workspace}/${flatpak_dir_shortname}":"/build-container-installer/${flatpak_dir_shortname}" \ + --volume "${workspace}"/scripts/files/output:/build-container-installer/build \ + ghcr.io/jasonn3/build-container-installer:main \ + ARCH="x86_64" \ + ENABLE_CACHE_DNF="false" \ + ENABLE_CACHE_SKOPEO="false" \ + ENABLE_FLATPAK_DEPENDENCIES="false" \ + ENROLLMENT_PASSWORD="ublue-os" \ + FLATPAK_REMOTE_REFS_DIR="${flatpak_dir_shortname}" \ + IMAGE_NAME="${tag}" \ + IMAGE_REPO="localhost" \ + IMAGE_TAG="${version}-${git_branch}" \ + ISO_NAME="build/${tag}-${version}-${git_branch}.iso" \ + SECURE_BOOT_KEY_URL='https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' \ + VARIANT="${variant}" \ + VERSION="${version}" diff --git a/scripts/build-iso.sh b/scripts/build-iso.sh index e98e329394e..e0f839b7cca 100755 --- a/scripts/build-iso.sh +++ b/scripts/build-iso.sh @@ -4,6 +4,9 @@ if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) +fi # Common Build ISO # shellcheck disable=SC1091 @@ -30,8 +33,8 @@ ${container_mgr} run --rm --privileged \ FLATPAK_REMOTE_REFS_DIR="${flatpak_dir_shortname}" \ IMAGE_NAME="${tag}" \ IMAGE_REPO="localhost" \ - IMAGE_TAG="${version}" \ - ISO_NAME="build/${tag}-${version}.iso" \ + IMAGE_TAG="${version}-${git_branch}" \ + ISO_NAME="build/${tag}-${version}-${git_branch}.iso" \ SECURE_BOOT_KEY_URL='https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' \ VARIANT="${variant}" \ VERSION="${version}" diff --git a/scripts/cleanup-dir.sh b/scripts/cleanup-dir.sh index c3116308ac1..741d8683348 100755 --- a/scripts/cleanup-dir.sh +++ b/scripts/cleanup-dir.sh @@ -2,11 +2,13 @@ if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi +# shellcheck disable=SC1091 +. "${project_root}/scripts/sudoif.sh" set -euox pipefail #shellcheck disable=SC2154 -rm -f "${project_root}"/scripts/files/output/* #ISOs +sudoif rm -f "${project_root}"/scripts/files/output/* #ISOs rm -f "${project_root}"/*_flatapks/flatpaks_with_deps #Flatpak Deps rm -rf "${project_root}"/flatpak.* #Flatpak Tempdir -rm -rf "${project_root}"/scripts/files/home/ublue-os/* #Test User Home +sudoif rm -rf "${project_root}"/scripts/files/home/ublue-os/* #Test User Home diff --git a/scripts/cleanup-images.sh b/scripts/cleanup-images.sh index 5875b040437..9b66d614b9d 100755 --- a/scripts/cleanup-images.sh +++ b/scripts/cleanup-images.sh @@ -8,7 +8,7 @@ container_mgr=( for i in "${container_mgr[@]}"; do if [[ $(command -v "$i") ]]; then echo "Container Manager: ${i}" - ID=$(${i} images --filter "reference=localhost/bluefin*-build" --filter "reference=localhost/aurora*-build" --format "{{.ID}}") + ID=$(${i} images --filter "reference=localhost/bluefin*-build*" --filter "reference=localhost/aurora*-build*" --format "{{.ID}}") xargs -I {} "${i}" image rm {} <<< "$ID" echo "" fi diff --git a/scripts/common-build-iso.sh b/scripts/common-build-iso.sh index 8eadb0a52c8..e4d507f582a 100644 --- a/scripts/common-build-iso.sh +++ b/scripts/common-build-iso.sh @@ -1,6 +1,9 @@ #!/usr/bin/bash #shellcheck disable=SC2154,SC2034 +# shellcheck disable=SC1091 +. "${project_root}/scripts/sudoif.sh" + # Check if inside rootless container if [[ -f /run/.containerenv ]]; then #shellcheck disable=SC1091 @@ -17,16 +20,6 @@ if "${container_mgr}" info | grep Root | grep -q /home; then echo "Cannot build ISO with rootless container..." exit 1 fi -function work-in-process(){ - echo "ISO Builder script is a Work In Process" - secs=5 - while [ $secs -gt 0 ] - do - printf "\r\033[KWaiting %.d seconds." $((secs--)) - sleep 1 - done -} -work-in-process # Get Inputs image=$1 @@ -43,6 +36,9 @@ tag=$(just _tag "${image}" "${target}") # Don't use -build suffix, flatpak dependency using ghcr ghcr_tag=${tag::-6} +# Remove old ISO if present +sudoif rm -f "${project_root}/scripts/files/output/${tag}-${version}-${git_branch}.iso" +sudoif rm -f "${project_root}/scripts/files/output/${tag}-${version}-${git_branch}.iso-CHECKSUM" # Set Base Image base_image=$(just _base_image "${image}") @@ -59,7 +55,7 @@ else fi # Make sure image actually exists, build if it doesn't -ID=$(${container_mgr} images --filter reference=localhost/"${tag}":"${version}" --format "{{.ID}}") +ID=$(${container_mgr} images --filter reference=localhost/"${tag}:${version}-${git_branch}" --format "{{.ID}}") if [[ -z ${ID} ]]; then just build "${image}" "${target}" "${version}" fi @@ -83,7 +79,7 @@ ostree refs --repo=\${FLATPAK_SYSTEM_DIR}/repo | grep '^deploy/' | grep -v 'org\ EOF workspace=${project_root} -if [[ -f /.dockerenv ]]; then +if [[ -f /.dockerenv || -f /run/.containerenv ]]; then FLATPAK_REFS_DIR=${LOCAL_WORKSPACE_FOLDER}/${flatpak_dir_shortname} TEMP_FLATPAK_INSTALL_DIR="${LOCAL_WORKSPACE_FOLDER}/$(echo "${TEMP_FLATPAK_INSTALL_DIR}" | rev | cut -d / -f 1 | rev)" workspace=${LOCAL_WORKSPACE_FOLDER} @@ -97,7 +93,7 @@ if [[ ! -f ${project_root}/${flatpak_dir_shortname}/flatpaks_with_deps ]]; then -e FLATPAK_TRIGGERSDIR=/flatpak/triggers \ --volume "${FLATPAK_REFS_DIR}":/output \ --volume "${TEMP_FLATPAK_INSTALL_DIR}":/temp_flatpak_install_dir \ - "ghcr.io/ublue-os/${ghcr_tag}:${version}" /temp_flatpak_install_dir/script.sh + "ghcr.io/ublue-os/${base_image}-main:${version}" /temp_flatpak_install_dir/script.sh fi # Remove Temp Directory @@ -105,7 +101,3 @@ if [[ -f /.dockerenv ]]; then TEMP_FLATPAK_INSTALL_DIR=${project_root}/$(echo "${TEMP_FLATPAK_INSTALL_DIR}" | rev | cut -d / -f 1 | rev) fi rm -rf "${TEMP_FLATPAK_INSTALL_DIR}" - -# Remove old ISO if present -rm -f "${project_root}/scripts/files/output/${tag}-${version}.iso" -rm -f "${project_root}/scripts/files/output/${tag}-${version}.iso-CHECKSUM" diff --git a/scripts/get-defaults.sh b/scripts/get-defaults.sh index b57af4fae41..36d811cd310 100755 --- a/scripts/get-defaults.sh +++ b/scripts/get-defaults.sh @@ -16,11 +16,16 @@ fi # if no target, base if [[ -z "${target}" ]]; then target="base" - # if $2 is numeric, assume that is version and target is base elif [[ ${target} =~ ^[0-9]+$ ]]; then version=${target} target="base" +elif [[ ${target} =~ latest ]]; then + version=${target} + target="base" +elif [[ ${target} =~ gts ]]; then + version=${target} + target="base" fi # if no version, bluefin is GTS, Aurora is Latest diff --git a/scripts/list-images.sh b/scripts/list-images.sh index 8c11ac44d98..ab527f69998 100755 --- a/scripts/list-images.sh +++ b/scripts/list-images.sh @@ -8,7 +8,7 @@ container_mgr=( for i in "${container_mgr[@]}"; do if [[ $(command -v "$i") ]]; then echo "Container Manager: ${i}" - ${i} images --filter "reference=localhost/bluefin*-build" --filter "reference=localhost/aurora*-build" + ${i} images --filter "reference=localhost/bluefin*-build*" --filter "reference=localhost/aurora*-build*" echo "" fi done diff --git a/scripts/run-booted-guest.sh b/scripts/run-booted-guest.sh index 08552f427c4..2d7abd12a32 100755 --- a/scripts/run-booted-guest.sh +++ b/scripts/run-booted-guest.sh @@ -2,6 +2,11 @@ if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) +fi +# shellcheck disable=SC2154,SC1091 +. "${project_root}/scripts/sudoif.sh" # Get inputs image=$1 @@ -28,7 +33,7 @@ if "${container_mgr}" info | grep Root | grep -q /home; then fi # Check to see if image exists, build it if it doesn't -ID=$(${container_mgr} images --filter reference=localhost/"${tag}:${version}" --format "{{.ID}}") +ID=$(${container_mgr} images --filter reference=localhost/"${tag}:${version}-${git_branch}" --format "{{.ID}}") if [[ -z ${ID} ]]; then just build "${image}" "${target}" "${version}" fi @@ -87,4 +92,4 @@ fi if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi -rm -rf "${project_root}/scripts/files/home/ublue-os" +sudoif rm -rf "${project_root}/scripts/files/home/ublue-os" diff --git a/scripts/run-booted-home.sh b/scripts/run-booted-home.sh index ff1fc6c71b4..44eac5ae6d2 100755 --- a/scripts/run-booted-home.sh +++ b/scripts/run-booted-home.sh @@ -2,6 +2,9 @@ if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) +fi # Get inputs image=$1 @@ -28,7 +31,7 @@ if "${container_mgr}" info | grep Root | grep -q /home; then fi # Check to see if image exists, build it if it doesn't -ID=$(${container_mgr} images --filter reference=localhost/"${tag}:${version}" --format "{{.ID}}") +ID=$(${container_mgr} images --filter reference=localhost/"${tag}:${version}-${git_branch}" --format "{{.ID}}") if [[ -z ${ID} ]]; then just build "${image}" "${target}" "${version}" fi diff --git a/scripts/run-image.sh b/scripts/run-image.sh index d86b582efae..c56aa650a43 100755 --- a/scripts/run-image.sh +++ b/scripts/run-image.sh @@ -2,6 +2,9 @@ if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) +fi set -eo pipefail # Get Inputs @@ -18,10 +21,10 @@ container_mgr=$(just _container_mgr) tag=$(just _tag "${image}" "${target}") # Check if requested image exist, if it doesn't build it -ID=$(${container_mgr} images --filter reference=localhost/"${tag}":"${version}" --format "{{.ID}}") +ID=$(${container_mgr} images --filter reference=localhost/"${tag}":"${version}-${git_branch}" --format "{{.ID}}") if [[ -z ${ID} ]]; then just build "${image}" "${target}" "${version}" fi # Run image -"${container_mgr}" run -it --rm localhost/"${tag}":"${version}" /usr/bin/bash +"${container_mgr}" run -it --rm localhost/"${tag}:${version}-${git_branch}" /usr/bin/bash diff --git a/scripts/run-iso.sh b/scripts/run-iso.sh index 8635f837e86..7edcd156f45 100755 --- a/scripts/run-iso.sh +++ b/scripts/run-iso.sh @@ -2,6 +2,9 @@ if [[ -z ${project_root} ]]; then project_root=$(git rev-parse --show-toplevel) fi +if [[ -z ${git_branch} ]]; then + git_branch=$(git branch --show-current) +fi set -eo pipefail # Get Inputs @@ -18,7 +21,7 @@ container_mgr=$(just _container_mgr) tag=$(just _tag "${image}" "${target}") #check if ISO exists. Create if it doesn't -if [[ ! -f "${project_root}/scripts/files/output/${tag}-${version}.iso" ]]; then +if [[ ! -f "${project_root}/scripts/files/output/${tag}-${version}-${git_branch}.iso" ]]; then just build-iso "$image" "$target" "$version" fi @@ -34,5 +37,5 @@ ${container_mgr} run --rm --cap-add NET_ADMIN \ --env "DISK_SIZE=64G" \ --env "BOOT_MODE=uefi" \ --device=/dev/kvm \ - --volume "${workspace}/scripts/files/output/${tag}-${version}.iso":/boot.iso \ + --volume "${workspace}/scripts/files/output/${tag}-${version}-${git_branch}.iso":/boot.iso \ docker.io/qemux/qemu-docker diff --git a/scripts/sudoif.sh b/scripts/sudoif.sh new file mode 100644 index 00000000000..aeb7c722972 --- /dev/null +++ b/scripts/sudoif.sh @@ -0,0 +1,14 @@ +#!/usr/bin/bash +function sudoif(){ + if [[ "${TERM_PROGRAM}" == "vscode" && \ + ! -f /run/.containerenv && \ + ! -f /.dockerenv ]]; then + [[ $(command -v systemd-run) ]] && \ + /usr/bin/systemd-run --uid=0 --gid=0 -d -E TERM="$TERM" -t -q -P -G "$@" \ + || exit 1 + else + [[ $(command -v sudo) ]] && \ + /usr/bin/sudo "$@" \ + || exit 1 + fi +}