Skip to content

Commit

Permalink
Use dataclass to define build configs
Browse files Browse the repository at this point in the history 
Signed-off-by: Nikolay Yurin <[email protected]>
  • Loading branch information
@yurinnick
yurinnick committed Dec 13, 2023
1 parent feda0f4 commit a690aa7
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 115 deletions.
256 changes: 145 additions & 111 deletions .github/scripts/matrix.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,25 @@
#!/usr/bin/env python3

from json import dumps
from enum import Enum
import os
import dataclasses
import json

from enum import Enum
from typing import Any, Dict, List, Final, Set, Union

MANAGED_OWNER: Final[str] = "kernel-patches"
MANAGED_REPOS: Final[Set[str]] = {
f"{MANAGED_OWNER}/bpf",
f"{MANAGED_OWNER}/vmtest",
}
# We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
# otherwise get library versioning issue such as
# `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
DEFAULT_RUNNER: Final[str] = "ubuntu-20.04"
DEFAULT_LLVM_VERSION: Final[int] = 17


class Arch(Enum):
class Arch(str, Enum):
"""
CPU architecture supported by CI.
"""
Expand All @@ -15,18 +29,102 @@ class Arch(Enum):
X86_64 = "x86_64"


class Compiler(str, Enum):
GCC = "gcc"
LLVM = "llvm"


@dataclasses.dataclass
class Toolchain:
compiler: Compiler
# This is relevant ONLY for LLVM and should not be required for GCC
version: int

@property
def short_name(self) -> str:
return str(self.compiler.value)

@property
def full_name(self) -> str:
if self.compiler == Compiler.GCC:
return self.short_name

return f"{self.short_name}-{self.version}"

def to_dict(self) -> Dict[str, Union[str, int]]:
return {
"name": self.short_name,
"fullname": self.full_name,
"version": self.version,
}


@dataclasses.dataclass
class BuildConfig:
arch: Arch
toolchain: Toolchain
kernel: str = "LATEST"
run_veristat: bool = False
parallel_tests: bool = False
build_release: bool = False

@property
def runs_on(self) -> List[str]:
if is_managed_repo():
return ["self-hosted", self.arch.value]
return [DEFAULT_RUNNER]

@property
def tests(self) -> Dict[str, Any]:
tests_list = [
"test_progs",
"test_progs_parallel",
"test_progs_no_alu32",
"test_progs_no_alu32_parallel",
"test_maps",
"test_verifier",
]

if self.toolchain.version >= 18:
tests_list.append("test_progs_cpuv4")

if not self.parallel_tests:
tests_list = [test for test in tests_list if not test.endswith("parallel")]

return {"include": [generate_test_config(test) for test in tests_list]}

def to_dict(self) -> Dict[str, Any]:
return {
"arch": self.arch.value,
"toolchain": self.toolchain.to_dict(),
"kernel": self.kernel,
"run_veristat": self.run_veristat,
"parallel_tests": self.parallel_tests,
"build_release": self.build_release,
"runs_on": self.runs_on,
"tests": self.tests,
}


def is_managed_repo() -> bool:
return (
os.environ["GITHUB_REPOSITORY_OWNER"] == MANAGED_OWNER
and os.environ["GITHUB_REPOSITORY"] in MANAGED_REPOS
)


def set_output(name, value):
"""Write an output variable to the GitHub output file."""
with open(os.getenv("GITHUB_OUTPUT"), "a", encoding="utf-8") as file:
file.write(f"{name}={value}\n")


def generate_test_config(test):
def generate_test_config(test: str) -> Dict[str, Union[str, int]]:
"""Create the configuration for the provided test."""
experimental = test.endswith("_parallel")
is_parallel = test.endswith("_parallel")
config = {
"test": test,
"continue_on_error": experimental,
"continue_on_error": is_parallel,
# While in experimental mode, parallel jobs may get stuck
# anywhere, including in user space where the kernel won't detect
# a problem and panic. We add a second layer of (smaller) timeouts
Expand All @@ -36,114 +134,50 @@ def generate_test_config(test):
# non-experimental jobs, 360 is the default which will be
# superseded by the overall workflow timeout (but we need to
# specify something).
"timeout_minutes": 30 if experimental else 360,
"timeout_minutes": 30 if is_parallel else 360,
}
return config


def get_tests(config):
tests = [
"test_progs",
"test_progs_parallel",
"test_progs_no_alu32",
"test_progs_no_alu32_parallel",
"test_maps",
"test_verifier",
if __name__ == "__main__":
matrix = [
BuildConfig(
arch=Arch.X86_64,
toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
run_veristat=True,
parallel_tests=True,
),
BuildConfig(
arch=Arch.X86_64,
toolchain=Toolchain(compiler=Compiler.LLVM, version=DEFAULT_LLVM_VERSION),
build_release=True,
),
BuildConfig(
arch=Arch.X86_64,
toolchain=Toolchain(compiler=Compiler.LLVM, version=18),
build_release=True,
),
BuildConfig(
arch=Arch.AARCH64,
toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
),
# BuildConfig(
# arch=Arch.AARCH64,
# toolchain=Toolchain(
# compiler=Compiler.LLVM,
# version=DEFAULT_LLVM_VERSION
# ),
# ),
BuildConfig(
arch=Arch.S390X,
toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
),
]
if int(config.get("llvm-version", 0)) >= 18:
tests.append("test_progs_cpuv4")
if config.get("parallel_tests", False):
return tests
return [test for test in tests if not test.endswith("parallel")]


matrix = [
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.X86_64.value,
"toolchain": "gcc",
"llvm-version": "17",
"run_veristat": True,
"parallel_tests": True,
},
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.X86_64.value,
"toolchain": "llvm",
"llvm-version": "17",
"build_release": True,
},
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.X86_64.value,
"toolchain": "llvm",
"llvm-version": "18",
"build_release": True,
},
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.AARCH64.value,
"toolchain": "gcc",
"llvm-version": "17",
},
# {
# "kernel": "LATEST",
# "runs_on": [],
# "arch": Arch.AARCH64.value,
# "toolchain": "llvm",
# "llvm-version": "16",
# },
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.S390X.value,
"toolchain": "gcc",
"llvm-version": "17",
},
]
self_hosted_repos = [
"kernel-patches/bpf",
"kernel-patches/vmtest",
]

for idx in range(len(matrix) - 1, -1, -1):
if matrix[idx]["toolchain"] == "gcc":
matrix[idx]["toolchain_full"] = "gcc"
else:
matrix[idx]["toolchain_full"] = "llvm-" + matrix[idx]["llvm-version"]

# Set run_veristat to false by default.
matrix[idx]["run_veristat"] = matrix[idx].get("run_veristat", False)
# Set build_release to false by default.
matrix[idx]["build_release"] = matrix[idx].get("build_release", False)
# Feel in the tests
matrix[idx]["tests"] = {
"include": [generate_test_config(test) for test in get_tests(matrix[idx])]
}

# Only a few repository within "kernel-patches" use self-hosted runners.
if (
os.environ["GITHUB_REPOSITORY_OWNER"] != "kernel-patches"
or os.environ["GITHUB_REPOSITORY"] not in self_hosted_repos
):
# Outside of those repositories, we only run on x86_64 GH hosted runners (ubuntu-20.04)
# We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
# otherwise get library versioning issue such as
# `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
for idx in range(len(matrix) - 1, -1, -1):
if matrix[idx]["arch"] != Arch.X86_64.value:
del matrix[idx]
else:
matrix[idx]["runs_on"] = ["ubuntu-20.04"]
else:
# Otherwise, run on (self-hosted, arch) runners
for idx in range(len(matrix) - 1, -1, -1):
matrix[idx]["runs_on"].extend(["self-hosted", matrix[idx]["arch"]])

build_matrix = {"include": matrix}

set_output("build_matrix", dumps(build_matrix))
# Outside of those repositories we only run on x86_64
if not is_managed_repo():
matrix = [config for config in matrix if config.arch == Arch.X86_64]

json_matrix = json.dumps({"include": [config.to_dict() for config in matrix]})
print(json_matrix)
set_output("build_matrix", json_matrix)
8 changes: 4 additions & 4 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
build-and-test:
# Setting name to arch-compiler here to avoid lengthy autogenerated names due to matrix
# e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
name: "${{ matrix.arch }}-${{ matrix.toolchain_full }}"
name: "${{ matrix.arch }}-${{ matrix.toolchain.fullname }}"
uses: ./.github/workflows/kernel-build-test.yml
needs: [set-matrix]
permissions:
Expand All @@ -41,10 +41,10 @@ jobs:
matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }}
with:
arch: ${{ matrix.arch }}
toolchain_full: ${{ matrix.toolchain_full }}
toolchain: ${{ matrix.toolchain }}
toolchain_full: ${{ matrix.toolchain.fullname }}
toolchain: ${{ matrix.toolchain.name }}
runs_on: ${{ toJSON(matrix.runs_on) }}
llvm-version: ${{ matrix.llvm-version }}
llvm-version: ${{ matrix.toolchain.version }}
kernel: ${{ matrix.kernel }}
tests: ${{ toJSON(matrix.tests) }}
run_veristat: ${{ matrix.run_veristat }}
Expand Down

0 comments on commit a690aa7

Please sign in to comment.