Refactor ebpf test runner (#208)

* init commit

* move around files

* go code done

* modify makefile, refactor reader

* final cleanup

* cleanup, format

* try newer go version?

* try different compile args?

* try to deal with weird buffer errors

* tinker with timeouts

* fix script

* tinker with scripts

* try to get timing info

* tinker with profiling

* fix resource leak

* bad merge

* re-add DNS test

* clean up test bins, improve dns tests

* format

* tinker with overlay tests

* add back removed test, small cleanup

* try to move around files

* rename, part one

* rename, part two

* restore old file

* try again

* try rename again

* try other command

* try restore

* restore, 2

* completely restore old file

* rename old file

* modify in different commit

* update docs

* clean up network tests

.github/workflows/multikernel-tester.yml

@@ -64,7 +64,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: '1.17'
+        go-version: '1.22'
     - name: Run tests
       run: make run-multikernel-test IMG_FILTER=${{ matrix.kernel_flavor }} ARCH=${{ inputs.architecture }} ARTIFACTS_PATH=${PWD}/artifacts
     - name: Prepare for archival

.gitignore

@@ -13,6 +13,7 @@ ctags
 cscope.out
 
 # Test results and other stuff in testing/
+testing/testrunner/testrunner.test
 testing/bpf-check-summary.txt
 testing/*.cpio
 testing/*.tar

Makefile

@@ -21,6 +21,9 @@ CONTAINER_LOCAL_TAG ?= ebpf-builder:${USER}-latest
 IMAGEPACK_REPOSITORY ?= ghcr.io/elastic/ebpf-imagepack
 IMAGEPACK_PULL_TAG ?= 20231006-0053
 
+TESTBIN_SRC = $(wildcard testing/test_bins/*.c)
+TESTBIN_PROGS = $(patsubst testing/test_bins/%.c,testing/test_bins/bin/${ARCH}/%,$(TESTBIN_SRC))
+
 ifdef BUILD_CONTAINER_IMAGE
 	CONTAINER_IMAGE = ${CONTAINER_LOCAL_TAG}
 else
@@ -170,5 +173,14 @@ endif
 	go install github.com/florianl/bluebox@b8590fb1850f56df6e6d7786931fcabdc1e9173d
 	cd testing && ./run_tests.sh ${ARCH} ${ARTIFACTS_PATH} ${PWD}/kernel-images/${IMG_FILTER}/${ARCH}/*
 
+testbins: testbinpath $(TESTBIN_PROGS)
+
+testbinpath: 
+	mkdir -p testing/test_bins/bin/${ARCH}
+
+testing/test_bins/bin/${ARCH}/%: testing/test_bins/%.c
+	$(CC) -g -static -o $@ $<
+
 clean:
 	${SUDO} rm -rf artifacts-*
+	rm -r testing/test_bins/bin/*

non-GPL/Events/EventsTrace/EventsTrace.c

@@ -280,7 +280,10 @@ static void out_escaped_string(const char *value)
             break;
         default:
             if (!isascii(c) || iscntrl(c))
-                printf("\\x%02x", c);
+                // \x is not a valid escape character in json,
+                // and something like '\xff' will break a remarkable number of JSON parsers.
+                // we have to print as '0xff'
+                printf("0x%02x", (uint8_t)c);
             else
                 printf("%c", c);
         }

testing/README.md

@@ -55,6 +55,22 @@ By default `run_tests.sh` will pass `-j$(nproc)` to `parallel` (i.e. spin up as
 many jobs as there are CPU cores). You can change this by passing
 `-j <number of jobs>` to `run-tests.sh`.
 
+## Running tests locally
+
+This test framework leverages the go stdlib test suite, so all ebpf tests can
+be run as normal go tests, outside of the bluebox VMs:
+
+```
+#compile the test in advance, so we don't need root to have a go environment
+go test -c
+# run all tests
+sudo ./testrunner.test
+# run a single test
+sudo ./testrunner.test -test.run TestEbpf/Tcpv6ConnectionClose
+# run in verbose mode 
+sudo ./testrunner.test -test.run TestEbpf/Tcpv6ConnectionClose -test.v
+```
+
 ## Building Kernels
 
 A dockerized setup is provided at `kernel_builder/` to build mainline kernel

testing/run_tests.sh

@@ -8,7 +8,7 @@
 readonly PROGNAME=$(basename $0)
 readonly ARGS="$@"
 
-readonly SUCCESS_STRING="ALL BPF TESTS PASSED"
+readonly SUCCESS_STRING="exit status 0"
 readonly SUMMARY_FILE="bpf-check-summary.txt"
 readonly RESULTS_DIR="results"
 
@@ -70,8 +70,7 @@ EOF
 }
 
 main() {
-    local arch=$1
-    local artifacts="$2"
+
     local jobs=$(nproc)
 
     while getopts "j:" opt; do
@@ -85,6 +84,9 @@ main() {
         esac
     done
 
+    local arch=$1
+    local artifacts="$2"
+
     shift 2
 
     is_empty $arch \

testing/scripts/gen_initramfs.sh

@@ -37,7 +37,7 @@ build_testrunner() {
     pushd testrunner > /dev/null
 
     go clean
-    GOARCH=$goarch go build
+    GOARCH=$goarch CGO_ENABLED=0 go test -c -o testrunner -ldflags '-extldflags "-static"'
 
     if [[ $? -ne 0 ]]
     then
@@ -78,7 +78,7 @@ invoke_bluebox() {
 
     local cmd="bluebox"
     cmd+=" -a $goarch"
-    cmd+=" -e testrunner/testrunner"
+    cmd+=" -e testrunner/testrunner:-test.v"
     cmd+=" -r $eventstrace"
     cmd+=" -r $tcfiltertests"
     cmd+=" -r $tcfilterbpf"

testing/test_bins/common.h

@@ -45,3 +45,20 @@ void gen_pid_info_json(char *buf, size_t size)
              (pid_t)syscall(SYS_gettid), getppid(), getpid(), getsid(0), getpgid(0), cap_permitted,
              cap_effective);
 }
+
+int dump_info(int client_port, int server_port)
+{
+    char pid_info[8192];
+    gen_pid_info_json(pid_info, sizeof(pid_info));
+
+    char netns[128];
+    ssize_t nbytes;
+    CHECK(nbytes = readlink("/proc/self/ns/net", netns, sizeof(netns)), -1);
+    netns[nbytes] = '\0';
+
+    uint64_t netns_inode;
+    sscanf(netns, "net:[%lu]", &netns_inode);
+
+    printf("{ \"pid_info\": %s, \"client_port\": %d, \"server_port\": %d, \"netns\": %lu }\n",
+           pid_info, client_port, server_port, netns_inode);
+}

testing/test_bins/create_rename_delete_file_container.c

@@ -135,6 +135,11 @@ int main()
 
 cleanup:
     // Clean up directories created by child
+
+    // in the 5.10 test kernels, this umount call fails, so don't check.
+    // Can't reproduce the issue locally, but `unmount` operations on overlayfs have historically
+    // been quirky.
+    umount2(ovl_mountpoint, MNT_FORCE);
     CHECK(rm_recursive(ovl_mountpoint), -1);
     CHECK(rm_recursive(ovl_upperdir), -1);
     CHECK(rm_recursive(ovl_lowerdir), -1);

testing/test_bins/tcpv4_connect.c

@@ -25,23 +25,6 @@
 
 #define BOUND_PORT 2048
 
-int dump_info(int client_port, int server_port)
-{
-    char pid_info[8192];
-    gen_pid_info_json(pid_info, sizeof(pid_info));
-
-    char netns[128];
-    ssize_t nbytes;
-    CHECK(nbytes = readlink("/proc/self/ns/net", netns, sizeof(netns)), -1);
-    netns[nbytes] = '\0';
-
-    uint64_t netns_inode;
-    sscanf(netns, "net:[%lu]", &netns_inode);
-
-    printf("{ \"pid_info\": %s, \"client_port\": %d, \"server_port\": %d, \"netns\": %lu }\n",
-           pid_info, client_port, server_port, netns_inode);
-}
-
 int main()
 {
     struct sockaddr_in serveraddr;

testing/test_bins/tcpv6_connect.c

@@ -25,23 +25,6 @@
 
 #define BOUND_PORT 2048
 
-int dump_info(int client_port, int server_port)
-{
-    char pid_info[8192];
-    gen_pid_info_json(pid_info, sizeof(pid_info));
-
-    char netns[128];
-    ssize_t nbytes;
-    CHECK(nbytes = readlink("/proc/self/ns/net", netns, sizeof(netns)), -1);
-    netns[nbytes] = '\0';
-
-    uint64_t netns_inode;
-    sscanf(netns, "net:[%lu]", &netns_inode);
-
-    printf("{ \"pid_info\": %s, \"client_port\": %d, \"server_port\": %d, \"netns\": %lu }\n",
-           pid_info, client_port, server_port, netns_inode);
-}
-
 int main()
 {
     struct sockaddr_in6 serveraddr;

testing/test_bins/udp_send.c

@@ -13,6 +13,8 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "common.h"
+
 int main(int argc, char *argv[])
 {
     struct sockaddr_in sin;
@@ -38,5 +40,7 @@ int main(int argc, char *argv[])
     else if (n != sizeof(buf))
         errx(1, "sendto: shortcount");
 
+    dump_info(53, 0);
+
     return (0);
 }