Skip to content

Commit

Permalink
Backports (v2-edge) (#600)
Browse files Browse the repository at this point in the history
  • Loading branch information
roosterfish authored Feb 4, 2025
2 parents cfdadca + b35d57f commit 33cb35e
Show file tree
Hide file tree
Showing 65 changed files with 3,366 additions and 2,003 deletions.
7 changes: 3 additions & 4 deletions .github/.jira_sync_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ settings:

# (Optional) Jira project components that should be attached to the created issue
# Component names are case-sensitive
# components:
# - LXD
# - MicroCloud
components:
- MicroCloud

# (Optional) GitHub labels. Only issues with one of those labels will be synchronized.
# If not specified, all issues will be synchronized
Expand All @@ -36,4 +35,4 @@ settings:
improvement: Story
feature: Story
investigation: Spike
bug: Bug
bug: Bug
159 changes: 159 additions & 0 deletions .github/actions/system-test/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
name: System test
inputs:
github_runner:
description: Whether or not this test is executed on a GitHub runner.
required: true
default: true
runs:
using: composite
steps:
- name: Tune disk performance
uses: canonical/lxd/.github/actions/tune-disk-performance@main

- name: Reclaim some space
uses: canonical/lxd/.github/actions/reclaim-disk-space@main

- name: Reclaim some memory
uses: canonical/lxd/.github/actions/reclaim-memory@main

- name: Disable Docker (GitHub runners)
if: inputs.github_runner == 'true'
uses: canonical/lxd/.github/actions/disable-docker@main

- name: "Disable br_netfilter"
shell: bash
run: |
# When br_netfilter is enabled, the multicast traffic that passes the native LXD bridge
# will get masqueraded too which breaks the multicast discovery.
sudo rmmod br_netfilter
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Install Go
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
with:
go-version-file: 'go.mod'

- name: Install dependencies
shell: bash
run: |
sudo add-apt-repository ppa:dqlite/dev -y --no-update
sudo apt-get update
sudo apt-get install --no-install-recommends -y libdqlite-dev pkg-config net-tools
- name: Download system test dependencies
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: system-test-deps
merge-multiple: true
# Pick the right home path of the runner user.
# The GitHub runners use /home/runner and the Canonical ones use /home/ubuntu.
# The path intput supports expansion.
path: ~/go/bin

- name: Make GOCOVERDIR
shell: bash
run: mkdir -p "${GOCOVERDIR}"
if: env.GOCOVERDIR != ''

- name: Sideload debug binaries
shell: bash
run: |
set -eux
# Binaries to sideload
export MICROCLOUD_DEBUG_PATH=~/go/bin/microcloud
export MICROCLOUDD_DEBUG_PATH=~/go/bin/microcloudd
echo "MICROCLOUD_DEBUG_PATH=${MICROCLOUD_DEBUG_PATH}" >> "${GITHUB_ENV}"
echo "MICROCLOUDD_DEBUG_PATH=${MICROCLOUDD_DEBUG_PATH}" >> "${GITHUB_ENV}"
- name: Strip debug binaries
if: env.GOCOVERDIR == ''
shell: bash
run: |
set -eux
strip -s "${MICROCLOUD_DEBUG_PATH}" "${MICROCLOUDD_DEBUG_PATH}"
- name: "Free up the ephemeral disk"
shell: bash
run: |
set -eux
if ! mountpoint --quiet /mnt; then
echo "INFO: no ephemeral disk mounted on /mnt"
mount
exit 0
fi
# If the rootfs and the ephemeral part are on the same physical disk, giving the whole
# disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
# runners to have a single disk, we immediately bail rather than trying to gracefully
# handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
# we will be able to stop worrying about that special case.
if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
lsblk
blkid
sudo fdisk -l
exit 1
fi
# Free-up the ephemeral disk to use it as storage device for LXD.
sudo swapoff /mnt/swapfile
ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
sudo umount /mnt
sudo wipefs -a "${ephemeral_disk}"
export TEST_STORAGE_SOURCE="${ephemeral_disk}"
echo "TEST_STORAGE_SOURCE=${TEST_STORAGE_SOURCE}" >> "${GITHUB_ENV}"
- name: "Setup host LXD"
shell: bash
run: |
set -eux
sudo snap install lxd --channel 5.21/edge
sudo lxd init --auto --storage-backend=zfs
# Save cached images into the (compressed) zpool
sudo lxc storage volume create default images
sudo lxc config set storage.images_volume=default/images
- name: "Prepare for system tests"
shell: bash
run: |
set -eux
chmod +x ~
export BASE_OS="${{ matrix.os }}"
export LXD_SNAP_CHANNEL="${{ matrix.lxd }}"
export MICROCEPH_SNAP_CHANNEL="${{ matrix.microceph }}"
export MICROOVN_SNAP_CHANNEL="${{ matrix.microovn }}"
export MICROCLOUD_SNAP_CHANNEL="${{ matrix.microcloud }}"
cd test
sudo --preserve-env=GOCOVERDIR,DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE,TESTBED_READY,BASE_OS,LXD_SNAP_CHANNEL,MICROCEPH_SNAP_CHANNEL,MICROOVN_SNAP_CHANNEL,MICROCLOUD_SNAP_CHANNEL ./main.sh setup
echo "TESTBED_READY=1" >> "${GITHUB_ENV}"
echo "BASE_OS=${BASE_OS}" >> "${GITHUB_ENV}"
echo "LXD_SNAP_CHANNEL=${LXD_SNAP_CHANNEL}" >> "${GITHUB_ENV}"
echo "MICROCEPH_SNAP_CHANNEL=${MICROCEPH_SNAP_CHANNEL}" >> "${GITHUB_ENV}"
echo "MICROOVN_SNAP_CHANNEL=${MICROOVN_SNAP_CHANNEL}" >> "${GITHUB_ENV}"
echo "MICROCLOUD_SNAP_CHANNEL=${MICROCLOUD_SNAP_CHANNEL}" >> "${GITHUB_ENV}"
- name: "Run system tests (${{ matrix.suite }})"
shell: bash
run: |
set -eux
chmod +x ~
cd test
sudo --preserve-env=GOCOVERDIR,DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE,TESTBED_READY,BASE_OS,LXD_SNAP_CHANNEL,MICROCEPH_SNAP_CHANNEL,MICROOVN_SNAP_CHANNEL,MICROCLOUD_SNAP_CHANNEL ./main.sh ${{ matrix.suite }}
echo "TIMESTAMP=$(date +%Y%m%d_%H%M%S_%N)" >> "${GITHUB_ENV}"
- name: Upload coverage data
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: coverage-${{ matrix.suite }}-${{ env.TIMESTAMP }}
path: ${{ env.GOCOVERDIR }}
if: env.GOCOVERDIR != ''
16 changes: 16 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,29 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
labels: []
schedule:
interval: "weekly"
target-branch: "main"

- package-ecosystem: "gomod"
directory: "/"
labels: []
schedule:
interval: "weekly"
target-branch: "main"

- package-ecosystem: "github-actions"
directory: "/"
labels: []
schedule:
interval: "weekly"
target-branch: "v2-edge"

- package-ecosystem: "gomod"
directory: "/"
labels: []
schedule:
interval: "weekly"
target-branch: "v2-edge"
4 changes: 4 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Documentation:
- changed-files:
- any-glob-to-any-file:
- doc/**/*
8 changes: 4 additions & 4 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
Expand All @@ -66,7 +66,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v3
uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1

# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
Expand All @@ -79,6 +79,6 @@ jobs:
# ./location_of_script_within_repo/buildscript.sh

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
with:
category: "/language:${{matrix.language}}"
34 changes: 19 additions & 15 deletions .github/workflows/security.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
if: ${{ github.ref_name == 'main' }}
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: main

Expand All @@ -35,12 +35,20 @@ jobs:
run: trivy fs --download-db-only --cache-dir /home/runner/vuln-cache
continue-on-error: true

- name: Use previous downloaded database
- name: Cache Trivy vulnerability database
if: ${{ steps.db_download.outcome == 'success' }}
uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
path: /home/runner/vuln-cache
key: trivy-cache-${{ github.run_id }}

- name: Use previously downloaded database instead
if: ${{ steps.db_download.outcome == 'failure' }}
uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
with:
path: /home/runner/vuln-cache
key: trivy-latest-cache
key: download-failed # Use a non existing key to fallback to restore-keys
restore-keys: trivy-cache-

- name: Run Trivy vulnerability scanner
run: |
Expand All @@ -51,15 +59,8 @@ jobs:
--severity LOW,MEDIUM,HIGH,CRITICAL \
--output trivy-microcloud-repo-scan-results.sarif .
- name: Cache Trivy vulnerability database
if: ${{ steps.db_download.outcome == 'success' }}
uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
with:
path: /home/runner/vuln-cache
key: trivy-latest-cache

- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v3
uses: github/codeql-action/upload-sarif@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
with:
sarif_file: "trivy-microcloud-repo-scan-results.sarif"
sha: ${{ github.sha }}
Expand All @@ -75,11 +76,13 @@ jobs:
include:
- channel: "3/edge"
branch: "main"
- channel: "2/candidate"
- channel: "2/stable"
branch: "v2-edge"
- channel: "1/stable"
branch: "v1-edge"
steps:
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ matrix.branch }}

Expand All @@ -90,7 +93,8 @@ jobs:
uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
with:
path: /home/runner/vuln-cache
key: trivy-latest-cache
key: download-failed # Use a non existing key to fallback to restore-keys
restore-keys: trivy-cache-

- name: Download snap for scan
run: |
Expand All @@ -112,7 +116,7 @@ jobs:
mv tmp.json snap-scan-results.sarif
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@f779452ac5af1c261dce0346a8f964149f49322b # v3.26.13
uses: github/codeql-action/upload-sarif@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
with:
sarif_file: "snap-scan-results.sarif"
sha: ${{ github.sha }}
Expand Down
Loading

0 comments on commit 33cb35e

Please sign in to comment.