From 9c8538181ed21afc3342d8fc2faf71f5c071cfab Mon Sep 17 00:00:00 2001 From: Jan Luebbe Date: Mon, 3 Feb 2025 07:14:05 +0100 Subject: [PATCH] .github/workflows: add beaglebone build and switch rpi to forrest runner I've not updated qemux86 yet due to the pending key handing changes. Signed-off-by: Jan Luebbe --- .github/workflows/generate.py | 188 +++++++++++++++----- .github/workflows/meta-rauc-beaglebone.yml | 101 +++++++++++ .github/workflows/meta-rauc-raspberrypi.yml | 82 +++++++-- 3 files changed, 318 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/meta-rauc-beaglebone.yml diff --git a/.github/workflows/generate.py b/.github/workflows/generate.py index c174d24..25d8a20 100755 --- a/.github/workflows/generate.py +++ b/.github/workflows/generate.py @@ -3,46 +3,62 @@ from jinja2 import Template TEMPLATE = """ -name: «« layer »» CI +name: build «« layer »» on: - # Trigger the workflow on push or pull request, - # but only for the master branch push: branches: - master paths: + - '.github/workflows/«« layer »».yml' - '«« layer »»/**' pull_request: branches: - master paths: + - '.github/workflows/«« layer »».yml' - '«« layer »»/**' + # allow rebuilding without a push + workflow_dispatch: {} + jobs: build: name: «« layer »» Build - runs-on: ubuntu-20.04 + # run on self-hosted runner for the main repo or if vars.BUILD_RUNS_ON is set + runs-on: >- + ${{ + (vars.BUILD_RUNS_ON != '' && fromJSON(vars.BUILD_RUNS_ON)) || + (github.repository == 'rauc/meta-rauc-community' && fromJSON('["self-hosted", "forrest", "build"]')) || + 'ubuntu-20.04' + }} steps: - name: Install required packages run: | - sudo apt-get install diffstat + sudo apt-get -q -y --no-install-recommends install diffstat tree - name: Checkout - uses: actions/checkout@v3 - «% for name, url in base_layers.items()|list + extra_layers.items()|list %» - - name: Clone «« name »» - run: git clone -b master «« url »» + uses: actions/checkout@v4 + «% for layer_name, layer_info in layers.items() %» + - name: Clone «« layer_name »» + run: git clone --shared --reference-if-able /srv/shared-git/«« layer_name »».git -b «« layer_info["branch"] | default(release) »» «« layer_info["repo"] »» «% endfor %» - name: Initialize build directory run: | source poky/oe-init-build-env build - bitbake-layers add-layer ../meta-rauc - «% for name in extra_layers.keys() %» - bitbake-layers add-layer ../«« name »» + «% for layer_path in add_layers %» + bitbake-layers add-layer ../«« layer_path »» «% endfor %» bitbake-layers add-layer ../«« layer »» - echo 'INHERIT += "rm_work"' >> conf/local.conf + if [ -f ~/.yocto/auto.conf ]; then + cp ~/.yocto/auto.conf conf/ + echo 'SOURCE_MIRROR_URL = "http://10.0.2.2/rauc-community/downloads"' >> conf/auto.conf + else + echo 'SSTATE_MIRRORS = "file://.* https://github-runner.pengutronix.de/rauc-community/sstate-cache/PATH"' >> conf/auto.conf + echo 'BB_SIGNATURE_HANDLER = "OEBasicHash"' >> conf/auto.conf + echo 'BB_HASHSERVE = ""' >> conf/auto.conf + echo 'OPKGBUILDCMD = "opkg-build -Z gzip -a -1n"' >> conf/auto.conf + echo 'INHERIT += "rm_work"' >> conf/auto.conf + fi echo 'DISTRO_FEATURES:remove = "alsa bluetooth usbgadget usbhost wifi nfs zeroconf pci 3g nfc x11 opengl ptest wayland vulkan"' >> conf/local.conf - echo 'SSTATE_MIRRORS = "file://.* http://195.201.147.117/sstate-cache/PATH"' >> conf/local.conf «% if machine %» echo 'MACHINE = "«« machine »»"' >> conf/local.conf «% endif %» @@ -53,60 +69,152 @@ «% for line in conf %» echo '«« line »»' >> conf/local.conf «% endfor %» - - name: bitbake parsing test + - name: Show configuration files + run: | + cd build/conf + rgrep . *.conf + - name: Test bitbake parsing run: | source poky/oe-init-build-env build bitbake -p + - name: Build rauc, rauc-native + run: | + source poky/oe-init-build-env build + bitbake rauc rauc-native + - name: Build «« image »» + run: | + source poky/oe-init-build-env build + bitbake «« image »» + - name: Build RAUC Bundle + run: | + source poky/oe-init-build-env build + bitbake «« bundle »» + - name: Cache Data + env: + CACHE_KEY: ${{ secrets.YOCTO_CACHE_KEY }} + if: ${{ env.CACHE_KEY }} + run: | + mkdir -p ~/.ssh + echo "$CACHE_KEY" >> ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -rvx --ignore-existing build/downloads rauc-community-cache: || true + rsync -rvx --ignore-existing build/sstate-cache rauc-community-cache: || true + - name: Show Artifacts + run: | + source poky/oe-init-build-env build + tree --du -h tmp/deploy/images || true + «% if artifacts %» + - name: Upload Artifacts + uses: jluebbe/forrest-upload-artifact@summary + with: + path: | + «% for artifact in artifacts %» + build/tmp/deploy/images/«« machine »»/«« artifact »» + «% endfor %» + «% endif %» """.lstrip() template = Template( - TEMPLATE, - block_start_string='«%', - block_end_string='%»', - variable_start_string='««', - variable_end_string='»»', - comment_start_string='«#', - comment_end_string='#»', - trim_blocks=True, - lstrip_blocks=True, - keep_trailing_newline=True, - ) + TEMPLATE, + block_start_string="«%", + block_end_string="%»", + variable_start_string="««", + variable_end_string="»»", + comment_start_string="«#", + comment_end_string="#»", + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True, +) -default_context = { - "base_layers": { - "poky": "git://git.yoctoproject.org/poky", - "meta-rauc": "https://github.com/rauc/meta-rauc.git" +default_layers = { + "poky": { + "repo": "https://github.com/yoctoproject/poky.git", + # added by default + "add": [], + }, + "meta-rauc": { + "repo": "https://github.com/rauc/meta-rauc.git", }, - "extra_layers": { +} + +default_context = { + "release": "master", + "layers": { + **default_layers, }, + "extra_layers": {}, + "add_layers": [], "machine": None, "conf": [], + "image": "core-image-minimal", + "bundle": "update-bundle", + "artifacts": [] } contexts = [ { + "layer": "meta-rauc-beaglebone", **default_context, - "layer": "meta-rauc-qemux86", - "fstypes": "tar.bz2 wic", - "wks_file": "qemux86-grub-efi.wks", + "release": "scarthgap", + "machine": "beaglebone-yocto", + "fstypes": "ext4 wic.zst", + "wks_file": "beaglebone-yocto-dual.wks.in", "conf": [ - 'EXTRA_IMAGEDEPENDS += "ovmf"', - 'PREFERRED_RPROVIDER_virtual-grub-bootconf = "rauc-qemu-grubconf"', + 'IMAGE_BOOT_FILES:append = " boot.scr"', + ], + "artifacts": [ + "core-image-minimal-beaglebone-yocto.rootfs.wic.xz", + "core-image-minimal-beaglebone-yocto.rootfs.spdx.tar.zst", + "update-bundle-beaglebone-yocto.raucb", ], }, + #{ + # "layer": "meta-rauc-qemux86", + # **default_context, + # "fstypes": "tar.bz2 wic.zst", + # "wks_file": "qemux86-grub-efi.wks", + # "conf": [ + # 'EXTRA_IMAGEDEPENDS += "ovmf"', + # 'PREFERRED_RPROVIDER_virtual-grub-bootconf = "rauc-qemu-grubconf"', + # ], + # "bundle": "qemu-demo-bundle", + # "artifacts": [ + # "core-image-minimal-qemux86-64.rootfs.wic.zst", + # ], + #}, { - **default_context, "layer": "meta-rauc-raspberrypi", - "extra_layers": { - "meta-raspberrypi": "git://git.yoctoproject.org/meta-raspberrypi", + **default_context, + "layers": { + **default_layers, + "meta-raspberrypi": { + "repo": "https://github.com/agherzan/meta-raspberrypi.git", + }, }, "machine": "raspberrypi4", - "fstypes": "ext4", + "fstypes": "ext4 wic.zst", "wks_file": "sdimage-dual-raspberrypi.wks.in", + "artifacts": [ + "core-image-minimal-raspberrypi4.rootfs.ext4", + "core-image-minimal-raspberrypi4.rootfs.manifest", + "core-image-minimal-raspberrypi4.rootfs.spdx.json", + "core-image-minimal-raspberrypi4.rootfs.testdata.json", + "update-bundle-raspberrypi4.raucb", + ], }, ] for context in contexts: + add_layers = context["add_layers"] = [] + for k, v in context["layers"].items(): + sub_layers = v.get("add") + if sub_layers is None: + add_layers.append(f"{k}") + else: + for add in sub_layers: + add_layers.append(f"{k}/{add}") + output = template.render(context) file_name = f"{context['layer']}.yml" with open(file_name, "w") as file: diff --git a/.github/workflows/meta-rauc-beaglebone.yml b/.github/workflows/meta-rauc-beaglebone.yml new file mode 100644 index 0000000..5d11644 --- /dev/null +++ b/.github/workflows/meta-rauc-beaglebone.yml @@ -0,0 +1,101 @@ +name: build meta-rauc-beaglebone + +on: + push: + branches: + - master + paths: + - '.github/workflows/meta-rauc-beaglebone.yml' + - 'meta-rauc-beaglebone/**' + pull_request: + branches: + - master + paths: + - '.github/workflows/meta-rauc-beaglebone.yml' + - 'meta-rauc-beaglebone/**' + # allow rebuilding without a push + workflow_dispatch: {} + +jobs: + build: + name: meta-rauc-beaglebone Build + # run on self-hosted runner for the main repo or if vars.BUILD_RUNS_ON is set + runs-on: >- + ${{ + (vars.BUILD_RUNS_ON != '' && fromJSON(vars.BUILD_RUNS_ON)) || + (github.repository == 'rauc/meta-rauc-community' && fromJSON('["self-hosted", "forrest", "build"]')) || + 'ubuntu-20.04' + }} + steps: + - name: Install required packages + run: | + sudo apt-get -q -y --no-install-recommends install diffstat tree + - name: Checkout + uses: actions/checkout@v4 + - name: Clone poky + run: git clone --shared --reference-if-able /srv/shared-git/poky.git -b scarthgap https://github.com/yoctoproject/poky.git + - name: Clone meta-rauc + run: git clone --shared --reference-if-able /srv/shared-git/meta-rauc.git -b scarthgap https://github.com/rauc/meta-rauc.git + - name: Initialize build directory + run: | + source poky/oe-init-build-env build + bitbake-layers add-layer ../meta-rauc + bitbake-layers add-layer ../meta-rauc-beaglebone + if [ -f ~/.yocto/auto.conf ]; then + cp ~/.yocto/auto.conf conf/ + echo 'SOURCE_MIRROR_URL = "http://10.0.2.2/rauc-community/downloads"' >> conf/auto.conf + else + echo 'SSTATE_MIRRORS = "file://.* https://github-runner.pengutronix.de/rauc-community/sstate-cache/PATH"' >> conf/auto.conf + echo 'BB_SIGNATURE_HANDLER = "OEBasicHash"' >> conf/auto.conf + echo 'BB_HASHSERVE = ""' >> conf/auto.conf + echo 'OPKGBUILDCMD = "opkg-build -Z gzip -a -1n"' >> conf/auto.conf + echo 'INHERIT += "rm_work"' >> conf/auto.conf + fi + echo 'DISTRO_FEATURES:remove = "alsa bluetooth usbgadget usbhost wifi nfs zeroconf pci 3g nfc x11 opengl ptest wayland vulkan"' >> conf/local.conf + echo 'MACHINE = "beaglebone-yocto"' >> conf/local.conf + echo 'DISTRO_FEATURES:append = " rauc"' >> conf/local.conf + echo 'IMAGE_INSTALL:append = " rauc"' >> conf/local.conf + echo 'IMAGE_FSTYPES = "ext4 wic.zst"' >> conf/local.conf + echo 'WKS_FILE = "beaglebone-yocto-dual.wks.in"' >> conf/local.conf + echo 'IMAGE_BOOT_FILES:append = " boot.scr"' >> conf/local.conf + - name: Show configuration files + run: | + cd build/conf + rgrep . *.conf + - name: Test bitbake parsing + run: | + source poky/oe-init-build-env build + bitbake -p + - name: Build rauc, rauc-native + run: | + source poky/oe-init-build-env build + bitbake rauc rauc-native + - name: Build core-image-minimal + run: | + source poky/oe-init-build-env build + bitbake core-image-minimal + - name: Build RAUC Bundle + run: | + source poky/oe-init-build-env build + bitbake update-bundle + - name: Cache Data + env: + CACHE_KEY: ${{ secrets.YOCTO_CACHE_KEY }} + if: ${{ env.CACHE_KEY }} + run: | + mkdir -p ~/.ssh + echo "$CACHE_KEY" >> ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -rvx --ignore-existing build/downloads rauc-community-cache: || true + rsync -rvx --ignore-existing build/sstate-cache rauc-community-cache: || true + - name: Show Artifacts + run: | + source poky/oe-init-build-env build + tree --du -h tmp/deploy/images || true + - name: Upload Artifacts + uses: jluebbe/forrest-upload-artifact@summary + with: + path: | + build/tmp/deploy/images/beaglebone-yocto/core-image-minimal-beaglebone-yocto.rootfs.wic.xz + build/tmp/deploy/images/beaglebone-yocto/core-image-minimal-beaglebone-yocto.rootfs.spdx.tar.zst + build/tmp/deploy/images/beaglebone-yocto/update-bundle-beaglebone-yocto.raucb diff --git a/.github/workflows/meta-rauc-raspberrypi.yml b/.github/workflows/meta-rauc-raspberrypi.yml index a62ebe1..581784e 100644 --- a/.github/workflows/meta-rauc-raspberrypi.yml +++ b/.github/workflows/meta-rauc-raspberrypi.yml @@ -1,49 +1,105 @@ -name: meta-rauc-raspberrypi CI +name: build meta-rauc-raspberrypi on: - # Trigger the workflow on push or pull request, - # but only for the master branch push: branches: - master paths: + - '.github/workflows/meta-rauc-raspberrypi.yml' - 'meta-rauc-raspberrypi/**' pull_request: branches: - master paths: + - '.github/workflows/meta-rauc-raspberrypi.yml' - 'meta-rauc-raspberrypi/**' + # allow rebuilding without a push + workflow_dispatch: {} + jobs: build: name: meta-rauc-raspberrypi Build - runs-on: ubuntu-20.04 + # run on self-hosted runner for the main repo or if vars.BUILD_RUNS_ON is set + runs-on: >- + ${{ + (vars.BUILD_RUNS_ON != '' && fromJSON(vars.BUILD_RUNS_ON)) || + (github.repository == 'rauc/meta-rauc-community' && fromJSON('["self-hosted", "forrest", "build"]')) || + 'ubuntu-20.04' + }} steps: - name: Install required packages run: | - sudo apt-get install diffstat + sudo apt-get -q -y --no-install-recommends install diffstat tree - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Clone poky - run: git clone -b master git://git.yoctoproject.org/poky + run: git clone --shared --reference-if-able /srv/shared-git/poky.git -b master https://github.com/yoctoproject/poky.git - name: Clone meta-rauc - run: git clone -b master https://github.com/rauc/meta-rauc.git + run: git clone --shared --reference-if-able /srv/shared-git/meta-rauc.git -b master https://github.com/rauc/meta-rauc.git - name: Clone meta-raspberrypi - run: git clone -b master git://git.yoctoproject.org/meta-raspberrypi + run: git clone --shared --reference-if-able /srv/shared-git/meta-raspberrypi.git -b master https://github.com/agherzan/meta-raspberrypi.git - name: Initialize build directory run: | source poky/oe-init-build-env build bitbake-layers add-layer ../meta-rauc bitbake-layers add-layer ../meta-raspberrypi bitbake-layers add-layer ../meta-rauc-raspberrypi - echo 'INHERIT += "rm_work"' >> conf/local.conf + if [ -f ~/.yocto/auto.conf ]; then + cp ~/.yocto/auto.conf conf/ + echo 'SOURCE_MIRROR_URL = "http://10.0.2.2/rauc-community/downloads"' >> conf/auto.conf + else + echo 'SSTATE_MIRRORS = "file://.* https://github-runner.pengutronix.de/rauc-community/sstate-cache/PATH"' >> conf/auto.conf + echo 'BB_SIGNATURE_HANDLER = "OEBasicHash"' >> conf/auto.conf + echo 'BB_HASHSERVE = ""' >> conf/auto.conf + echo 'OPKGBUILDCMD = "opkg-build -Z gzip -a -1n"' >> conf/auto.conf + echo 'INHERIT += "rm_work"' >> conf/auto.conf + fi echo 'DISTRO_FEATURES:remove = "alsa bluetooth usbgadget usbhost wifi nfs zeroconf pci 3g nfc x11 opengl ptest wayland vulkan"' >> conf/local.conf - echo 'SSTATE_MIRRORS = "file://.* http://195.201.147.117/sstate-cache/PATH"' >> conf/local.conf echo 'MACHINE = "raspberrypi4"' >> conf/local.conf echo 'DISTRO_FEATURES:append = " rauc"' >> conf/local.conf echo 'IMAGE_INSTALL:append = " rauc"' >> conf/local.conf - echo 'IMAGE_FSTYPES = "ext4"' >> conf/local.conf + echo 'IMAGE_FSTYPES = "ext4 wic.zst"' >> conf/local.conf echo 'WKS_FILE = "sdimage-dual-raspberrypi.wks.in"' >> conf/local.conf - - name: bitbake parsing test + - name: Show configuration files + run: | + cd build/conf + rgrep . *.conf + - name: Test bitbake parsing run: | source poky/oe-init-build-env build bitbake -p + - name: Build rauc, rauc-native + run: | + source poky/oe-init-build-env build + bitbake rauc rauc-native + - name: Build core-image-minimal + run: | + source poky/oe-init-build-env build + bitbake core-image-minimal + - name: Build RAUC Bundle + run: | + source poky/oe-init-build-env build + bitbake update-bundle + - name: Cache Data + env: + CACHE_KEY: ${{ secrets.YOCTO_CACHE_KEY }} + if: ${{ env.CACHE_KEY }} + run: | + mkdir -p ~/.ssh + echo "$CACHE_KEY" >> ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -rvx --ignore-existing build/downloads rauc-community-cache: || true + rsync -rvx --ignore-existing build/sstate-cache rauc-community-cache: || true + - name: Show Artifacts + run: | + source poky/oe-init-build-env build + tree --du -h tmp/deploy/images || true + - name: Upload Artifacts + uses: jluebbe/forrest-upload-artifact@summary + with: + path: | + build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.ext4 + build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.manifest + build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.spdx.json + build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.testdata.json + build/tmp/deploy/images/raspberrypi4/update-bundle-raspberrypi4.raucb