tests(nss/testutils): Update coverage generation and merging

.github/workflows/qa.yaml

@@ -90,17 +90,33 @@ jobs:
         run: |
           set -eu
           cargo install grcov
-      - name: Run tests
+      - name: Run tests (with coverage collection)
         run: |
           set -eu
 
-          go test -coverpkg=./... -coverprofile=/tmp/coverage.out -covermode=set ./... -shuffle=on
+          # The coverage is not written if the output directory does not exist, so we need to create it.
+          raw_cov_dir="/tmp/raw_files"
+          rm -fr "${raw_cov_dir}"
+          mkdir -p "${raw_cov_dir}"
+
+          # Overriding the default coverage directory is not an exported flag of go test (yet), so
+          # we need to override it using the test.gocoverdir flag instead.
+          #TODO: Update when https://go-review.googlesource.com/c/go/+/456595 is merged.
+          go test -cover -covermode=set ./... -shuffle=on -args -test.gocoverdir="${raw_cov_dir}"
+
+          # Convert the raw coverage data into textfmt so we can merge the Rust one into it
+          go tool covdata textfmt -i="${raw_cov_dir}" -o="/tmp/coverage.out"
+
+          # Append the Rust coverage data to the Go one
+          cat "${raw_cov_dir}/rust-cov/rust2go_coverage" >>"/tmp/coverage.out"
+
+          # Filter out the testutils package and the pb.go file
+          grep -v -e "testutils" -e "pb.go" "/tmp/coverage.out" >"/tmp/coverage.out.filtered"
+
       - name: Run tests (with race detector)
         run: |
           go test -race ./...
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          file: /tmp/coverage.combined
-
-  # TODO: rust-tests:
+          file: /tmp/coverage.out.filtered

gotestcov

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# This is a wrapper script to run the Go tests and generate the coverage report.
+# The coverage will then be merged with the Rust one and the HTML version will be
+# exposed on localhost:6061.
+
+set -eu
+
+# find_go_mod walks up the directory tree looking for the go.mod file.
+# If it doesn't find it, the script will be aborted.
+find_go_mod() {
+    cwd="$(pwd)"
+
+    while [ "$cwd" != "/" ]; do
+        if [ -f "$cwd/go.mod" ]; then
+            echo "$cwd"
+            return
+        fi
+        cwd=$(dirname "$cwd")
+    done
+    echo "Error: go.mod not found in parent path. Aborting!"
+    exit 1
+}
+
+projectroot="$(find_go_mod)"
+cov_dir="${projectroot}/coverage"
+mkdir -p "${cov_dir}"
+
+# start http server on 6061 if none
+if ! $(nc -z localhost 6061); then
+    nohup python3 -m http.server --directory "${cov_dir}" 6061 1>/dev/null 2>&1 &
+fi
+
+raw_cov_dir="${cov_dir}/raw_files"
+
+rm -fr "${raw_cov_dir}"
+mkdir -p "${raw_cov_dir}"
+
+# Run the tests adding the necessary flags to enable coverage
+# Overriding the default coverage directory is currently not an exported flag of go test
+# We need to override it using the test.gocoverdir flag instead.
+#TODO: Update when https://go-review.googlesource.com/c/go/+/456595 is merged.
+go test -cover -covermode=set $@ -shuffle=on -args -test.gocoverdir="${raw_cov_dir}"
+
+# Convert the raw coverage data into textfmt so we can merge the Rust one into it
+go tool covdata textfmt -i="${raw_cov_dir}" -o="${cov_dir}/coverage.out"
+
+# Append the Rust coverage data to the Go one
+cat "${raw_cov_dir}/rust-cov/rust2go_coverage" >>"${cov_dir}/coverage.out"
+
+# Filter out the testutils package and the pb.go file
+grep -v -e "testutils" -e "pb.go" "${cov_dir}/coverage.out" >"${cov_dir}/coverage.out.filtered"
+
+# Generate the HTML report
+go tool cover -o "${cov_dir}/index.html" -html="${cov_dir}/coverage.out.filtered"

internal/testutils/coverage.go

@@ -2,7 +2,6 @@ package testutils
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -13,110 +12,13 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/exp/slices"
 )
 
 var (
-	goMainCoverProfile     string
-	goMainCoverProfileOnce sync.Once
-
-	coveragesToMerge   []string
-	coveragesToMergeMu sync.Mutex
+	goCoverDir     string
+	goCoverDirOnce sync.Once
 )
 
-// TrackTestCoverage starts tracking coverage in a dedicated file based on current test name.
-// This file will be merged to the current coverage main file.
-// It’s up to the test use the returned path to file golang-compatible cover format content.
-// To collect all coverages, then MergeCoverages() should be called after m.Run().
-// If coverage is not enabled, nothing is done.
-func TrackTestCoverage(t *testing.T) (testCoverFile string) {
-	t.Helper()
-
-	coverProfile := CoverProfile()
-	if coverProfile == "" {
-		return ""
-	}
-
-	coverAbsPath, err := filepath.Abs(coverProfile)
-	require.NoError(t, err, "Setup: can't transform go cover profile to absolute path")
-
-	testCoverFile = fmt.Sprintf("%s.%s", coverAbsPath, strings.ReplaceAll(
-		strings.ReplaceAll(t.Name(), "/", "_"),
-		"\\", "_"))
-
-	MarkCoverageForMerging(testCoverFile)
-
-	return testCoverFile
-}
-
-// MarkCoverageForMerging marks the coverage file to be merged to the main go cover profile.
-func MarkCoverageForMerging(coverFile string) {
-	coveragesToMergeMu.Lock()
-	defer coveragesToMergeMu.Unlock()
-	if slices.Contains(coveragesToMerge, coverFile) {
-		panic(fmt.Sprintf("Trying to adding a second time %q to the list of file to cover. This will create some overwrite and thus, should be only called once", coverFile))
-	}
-	coveragesToMerge = append(coveragesToMerge, coverFile)
-}
-
-// MergeCoverages append all coverage files marked for merging to main Go Cover Profile.
-// This has to be called after m.Run() in TestMain so that the main go cover profile is created.
-// This has no action if profiling is not enabled.
-func MergeCoverages() error {
-	coveragesToMergeMu.Lock()
-	defer coveragesToMergeMu.Unlock()
-	var err error
-	for _, cov := range coveragesToMerge {
-		err = errors.Join(err, appendToFile(cov, goMainCoverProfile))
-	}
-
-	if err != nil {
-		err = fmt.Errorf("teardown: couldn't inject coverages into the golang one: %w", err)
-	}
-	coveragesToMerge = nil
-	return err
-}
-
-// WantCoverage returns true if coverage was requested in test.
-func WantCoverage() bool {
-	for _, arg := range os.Args {
-		if !strings.HasPrefix(arg, "-test.coverprofile=") {
-			continue
-		}
-		return true
-	}
-	return false
-}
-
-// appendToFile appends src to the dst coverprofile file at the end.
-func appendToFile(src, dst string) error {
-	f, err := os.Open(filepath.Clean(src))
-	if err != nil {
-		return fmt.Errorf("can't open coverage file: %w", err)
-	}
-	defer f.Close()
-
-	d, err := os.OpenFile(dst, os.O_APPEND|os.O_WRONLY, 0600)
-	if err != nil {
-		return fmt.Errorf("can't open golang cover profile file: %w", err)
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		if strings.HasPrefix(scanner.Text(), "mode: ") {
-			continue
-		}
-		if _, err := d.Write([]byte(scanner.Text() + "\n")); err != nil {
-			return fmt.Errorf("can't write to golang cover profile file: %w", err)
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("error while scanning golang cover profile file: %w", err)
-	}
-	return nil
-}
-
 // fqdnToPath allows to return the fqdn path for this file relative to go.mod.
 func fqdnToPath(t *testing.T, path string) string {
 	t.Helper()
@@ -149,17 +51,25 @@ func fqdnToPath(t *testing.T, path string) string {
 	return ""
 }
 
-// CoverProfile parses the arguments to find the cover profile file.
-func CoverProfile() string {
-	goMainCoverProfileOnce.Do(func() {
+// AppendCovEnv returns the env needed to enable coverage when running a go binary.
+func AppendCovEnv(env []string) []string {
+	if CoverDir() == "" {
+		return env
+	}
+	return append(env, fmt.Sprintf("GOCOVERDIR=%s", CoverDir()))
+}
+
+// CoverDir parses the arguments to find the cover profile file.
+func CoverDir() string {
+	goCoverDirOnce.Do(func() {
 		for _, arg := range os.Args {
-			if !strings.HasPrefix(arg, "-test.coverprofile=") {
+			if !strings.HasPrefix(arg, "-test.gocoverdir=") {
 				continue
 			}
-			goMainCoverProfile = strings.TrimPrefix(arg, "-test.coverprofile=")
+			goCoverDir = strings.TrimPrefix(arg, "-test.gocoverdir=")
 		}
 	})
-	return goMainCoverProfile
+	return goCoverDir
 }
 
 // writeGoCoverageLine writes given line in go coverage format to w.

internal/testutils/rust.go

@@ -15,18 +15,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// MarkRustFilesForTestCache marks all rust files and related content to be in the Go test caching infra.
-func MarkRustFilesForTestCache(t *testing.T, rustDir string) {
-	t.Helper()
-
-	markForTestCache(t, []string{
-		filepath.Join(rustDir, "src"),
-		filepath.Join(rustDir, "testdata"),
-		filepath.Join(rustDir, "Cargo.toml"),
-		filepath.Join(rustDir, "Cargo.lock"),
-	})
-}
-
 // CanRunRustTests returns if we can run rust tests via cargo on this machine.
 // It checks for code coverage report if supported.
 func CanRunRustTests(coverageWanted bool) (err error) {
@@ -78,18 +66,13 @@ func TrackRustCoverage(t *testing.T, src string) (env []string, target string) {
 		target = t.TempDir()
 	}
 
-	testGoCoverage := TrackTestCoverage(t)
-	if testGoCoverage == "" {
-		return []string{}, target
-	}
-
-	coverDir := filepath.Dir(testGoCoverage)
-	if c := os.Getenv("RAW_COVER_DIR"); c != "" {
-		coverDir = c
+	coverDir := filepath.Join(CoverDir(), "rust-cov")
+	if coverDir == "" {
+		return nil, target
 	}
 
 	t.Cleanup(func() {
-		rustJSONCoverage := testGoCoverage + ".json"
+		rustJSONCoverage := filepath.Join(coverDir, "rust_coverage.json")
 		//nolint:gosec // G204 we define what we cover ourself
 		cmd := exec.Command("grcov", coverDir,
 			"--binary-path", filepath.Join(target, "debug"),
@@ -113,7 +96,7 @@ func TrackRustCoverage(t *testing.T, src string) (env []string, target string) {
 		require.NoError(t, err, "Teardown: decode our json coverage file")
 
 		// This is the destination file for rust coverage in go format.
-		outF, err := os.Create(testGoCoverage)
+		outF, err := os.Create(filepath.Join(coverDir, "rust2go_coverage"))
 		require.NoErrorf(t, err, "Teardown: failed opening output golang compatible cover file: %s", err)
 		defer func() { assert.NoError(t, outF.Close(), "Teardown: can’t close golang compatible cover file") }()
 
@@ -189,15 +172,3 @@ func convertRustFileResult(t *testing.T, results []interface{}, p string, w io.W
 		writeGoCoverageLine(t, w, p, l+1, 9999, covered)
 	}
 }
-
-// markForTestCache list all root directories/files so that they are marked in the test cache.
-func markForTestCache(t *testing.T, roots []string) {
-	t.Helper()
-
-	for _, root := range roots {
-		err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
-			return nil
-		})
-		require.NoError(t, err, "Setup: Error when listing input files for caching handling")
-	}
-}

nss/integration-tests/helper_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/ubuntu/authd/internal/testutils"
 )
 
-var daemonPath, rawCovDir string
+var daemonPath string
 
 // buildRustNSSLib builds the NSS library and links the compiled file to libPath.
 func buildRustNSSLib(t *testing.T) {
@@ -32,9 +32,8 @@ func buildRustNSSLib(t *testing.T) {
 		cargo = "cargo"
 	}
 
-	rustDir := filepath.Join(projectRoot, "nss")
-	testutils.MarkRustFilesForTestCache(t, rustDir)
 	var target string
+	rustDir := filepath.Join(projectRoot, "nss")
 	rustCovEnv, target = testutils.TrackRustCoverage(t, rustDir)
 
 	// Builds the nss library.
@@ -115,12 +114,7 @@ paths:
 	// #nosec:G204 - we control the command arguments in tests
 	cmd := exec.Command(daemonPath, "-c", configPath)
 	cmd.Stderr = os.Stderr
-	if testutils.WantCoverage() {
-		if rawCovDir == "" {
-			t.Fatalf("Setup: coverage is wanted but no coverage dir was set")
-		}
-		cmd.Env = append(cmd.Env, fmt.Sprintf("GOCOVERDIR=%s", rawCovDir))
-	}
+	cmd.Env = testutils.AppendCovEnv(cmd.Env)
 
 	stopped = make(chan struct{})
 	go func() {
@@ -152,7 +146,7 @@ func buildDaemon() (execPath string, cleanup func(), err error) {
 	execPath = filepath.Join(tempDir, "authd")
 	cmd := exec.Command("go", "build")
 	cmd.Dir = projectRoot
-	if testutils.WantCoverage() {
+	if testutils.CoverDir() != "" {
 		// -cover is a "positional flag", so it needs to come right after the "build" command.
 		cmd.Args = append(cmd.Args, "-cover")
 	}