diff --git a/.github/workflows/qa.yaml b/.github/workflows/qa.yaml index 7c6ee1b2f..1eb4e83d0 100644 --- a/.github/workflows/qa.yaml +++ b/.github/workflows/qa.yaml @@ -90,17 +90,33 @@ jobs: run: | set -eu cargo install grcov - - name: Run tests + - name: Run tests (with coverage collection) run: | set -eu - go test -coverpkg=./... -coverprofile=/tmp/coverage.out -covermode=set ./... -shuffle=on + # The coverage is not written if the output directory does not exist, so we need to create it. + raw_cov_dir="/tmp/raw_files" + rm -fr "${raw_cov_dir}" + mkdir -p "${raw_cov_dir}" + + # Overriding the default coverage directory is not an exported flag of go test (yet), so + # we need to override it using the test.gocoverdir flag instead. + #TODO: Update when https://go-review.googlesource.com/c/go/+/456595 is merged. + go test -cover -covermode=set ./... -shuffle=on -args -test.gocoverdir="${raw_cov_dir}" + + # Convert the raw coverage data into textfmt so we can merge the Rust one into it + go tool covdata textfmt -i="${raw_cov_dir}" -o="/tmp/coverage.out" + + # Append the Rust coverage data to the Go one + cat "${raw_cov_dir}/rust-cov/rust2go_coverage" >>"/tmp/coverage.out" + + # Filter out the testutils package and the pb.go file + grep -v -e "testutils" -e "pb.go" "/tmp/coverage.out" >"/tmp/coverage.out.filtered" + - name: Run tests (with race detector) run: | go test -race ./... - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - file: /tmp/coverage.combined - - # TODO: rust-tests: + file: /tmp/coverage.out.filtered diff --git a/gotestcov b/gotestcov new file mode 100755 index 000000000..4b64816dc --- /dev/null +++ b/gotestcov @@ -0,0 +1,55 @@ +#!/bin/bash + +# This is a wrapper script to run the Go tests and generate the coverage report. +# The coverage will then be merged with the Rust one and the HTML version will be +# exposed on localhost:6061. + +set -eu + +# find_go_mod walks up the directory tree looking for the go.mod file. +# If it doesn't find it, the script will be aborted. +find_go_mod() { + cwd="$(pwd)" + + while [ "$cwd" != "/" ]; do + if [ -f "$cwd/go.mod" ]; then + echo "$cwd" + return + fi + cwd=$(dirname "$cwd") + done + echo "Error: go.mod not found in parent path. Aborting!" + exit 1 +} + +projectroot="$(find_go_mod)" +cov_dir="${projectroot}/coverage" +mkdir -p "${cov_dir}" + +# start http server on 6061 if none +if ! $(nc -z localhost 6061); then + nohup python3 -m http.server --directory "${cov_dir}" 6061 1>/dev/null 2>&1 & +fi + +raw_cov_dir="${cov_dir}/raw_files" + +rm -fr "${raw_cov_dir}" +mkdir -p "${raw_cov_dir}" + +# Run the tests adding the necessary flags to enable coverage +# Overriding the default coverage directory is currently not an exported flag of go test +# We need to override it using the test.gocoverdir flag instead. +#TODO: Update when https://go-review.googlesource.com/c/go/+/456595 is merged. +go test -cover -covermode=set $@ -shuffle=on -args -test.gocoverdir="${raw_cov_dir}" + +# Convert the raw coverage data into textfmt so we can merge the Rust one into it +go tool covdata textfmt -i="${raw_cov_dir}" -o="${cov_dir}/coverage.out" + +# Append the Rust coverage data to the Go one +cat "${raw_cov_dir}/rust-cov/rust2go_coverage" >>"${cov_dir}/coverage.out" + +# Filter out the testutils package and the pb.go file +grep -v -e "testutils" -e "pb.go" "${cov_dir}/coverage.out" >"${cov_dir}/coverage.out.filtered" + +# Generate the HTML report +go tool cover -o "${cov_dir}/index.html" -html="${cov_dir}/coverage.out.filtered" diff --git a/internal/testutils/coverage.go b/internal/testutils/coverage.go index 4d1536387..48f02febe 100644 --- a/internal/testutils/coverage.go +++ b/internal/testutils/coverage.go @@ -2,7 +2,6 @@ package testutils import ( "bufio" - "errors" "fmt" "io" "os" @@ -13,110 +12,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" ) var ( - goMainCoverProfile string - goMainCoverProfileOnce sync.Once - - coveragesToMerge []string - coveragesToMergeMu sync.Mutex + goCoverDir string + goCoverDirOnce sync.Once ) -// TrackTestCoverage starts tracking coverage in a dedicated file based on current test name. -// This file will be merged to the current coverage main file. -// It’s up to the test use the returned path to file golang-compatible cover format content. -// To collect all coverages, then MergeCoverages() should be called after m.Run(). -// If coverage is not enabled, nothing is done. -func TrackTestCoverage(t *testing.T) (testCoverFile string) { - t.Helper() - - coverProfile := CoverProfile() - if coverProfile == "" { - return "" - } - - coverAbsPath, err := filepath.Abs(coverProfile) - require.NoError(t, err, "Setup: can't transform go cover profile to absolute path") - - testCoverFile = fmt.Sprintf("%s.%s", coverAbsPath, strings.ReplaceAll( - strings.ReplaceAll(t.Name(), "/", "_"), - "\\", "_")) - - MarkCoverageForMerging(testCoverFile) - - return testCoverFile -} - -// MarkCoverageForMerging marks the coverage file to be merged to the main go cover profile. -func MarkCoverageForMerging(coverFile string) { - coveragesToMergeMu.Lock() - defer coveragesToMergeMu.Unlock() - if slices.Contains(coveragesToMerge, coverFile) { - panic(fmt.Sprintf("Trying to adding a second time %q to the list of file to cover. This will create some overwrite and thus, should be only called once", coverFile)) - } - coveragesToMerge = append(coveragesToMerge, coverFile) -} - -// MergeCoverages append all coverage files marked for merging to main Go Cover Profile. -// This has to be called after m.Run() in TestMain so that the main go cover profile is created. -// This has no action if profiling is not enabled. -func MergeCoverages() error { - coveragesToMergeMu.Lock() - defer coveragesToMergeMu.Unlock() - var err error - for _, cov := range coveragesToMerge { - err = errors.Join(err, appendToFile(cov, goMainCoverProfile)) - } - - if err != nil { - err = fmt.Errorf("teardown: couldn't inject coverages into the golang one: %w", err) - } - coveragesToMerge = nil - return err -} - -// WantCoverage returns true if coverage was requested in test. -func WantCoverage() bool { - for _, arg := range os.Args { - if !strings.HasPrefix(arg, "-test.coverprofile=") { - continue - } - return true - } - return false -} - -// appendToFile appends src to the dst coverprofile file at the end. -func appendToFile(src, dst string) error { - f, err := os.Open(filepath.Clean(src)) - if err != nil { - return fmt.Errorf("can't open coverage file: %w", err) - } - defer f.Close() - - d, err := os.OpenFile(dst, os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - return fmt.Errorf("can't open golang cover profile file: %w", err) - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - if strings.HasPrefix(scanner.Text(), "mode: ") { - continue - } - if _, err := d.Write([]byte(scanner.Text() + "\n")); err != nil { - return fmt.Errorf("can't write to golang cover profile file: %w", err) - } - } - if err := scanner.Err(); err != nil { - return fmt.Errorf("error while scanning golang cover profile file: %w", err) - } - return nil -} - // fqdnToPath allows to return the fqdn path for this file relative to go.mod. func fqdnToPath(t *testing.T, path string) string { t.Helper() @@ -149,17 +51,25 @@ func fqdnToPath(t *testing.T, path string) string { return "" } -// CoverProfile parses the arguments to find the cover profile file. -func CoverProfile() string { - goMainCoverProfileOnce.Do(func() { +// AppendCovEnv returns the env needed to enable coverage when running a go binary. +func AppendCovEnv(env []string) []string { + if CoverDir() == "" { + return env + } + return append(env, fmt.Sprintf("GOCOVERDIR=%s", CoverDir())) +} + +// CoverDir parses the arguments to find the cover profile file. +func CoverDir() string { + goCoverDirOnce.Do(func() { for _, arg := range os.Args { - if !strings.HasPrefix(arg, "-test.coverprofile=") { + if !strings.HasPrefix(arg, "-test.gocoverdir=") { continue } - goMainCoverProfile = strings.TrimPrefix(arg, "-test.coverprofile=") + goCoverDir = strings.TrimPrefix(arg, "-test.gocoverdir=") } }) - return goMainCoverProfile + return goCoverDir } // writeGoCoverageLine writes given line in go coverage format to w. diff --git a/internal/testutils/rust.go b/internal/testutils/rust.go index d2bdc7636..b35e5a93b 100644 --- a/internal/testutils/rust.go +++ b/internal/testutils/rust.go @@ -15,18 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// MarkRustFilesForTestCache marks all rust files and related content to be in the Go test caching infra. -func MarkRustFilesForTestCache(t *testing.T, rustDir string) { - t.Helper() - - markForTestCache(t, []string{ - filepath.Join(rustDir, "src"), - filepath.Join(rustDir, "testdata"), - filepath.Join(rustDir, "Cargo.toml"), - filepath.Join(rustDir, "Cargo.lock"), - }) -} - // CanRunRustTests returns if we can run rust tests via cargo on this machine. // It checks for code coverage report if supported. func CanRunRustTests(coverageWanted bool) (err error) { @@ -78,18 +66,13 @@ func TrackRustCoverage(t *testing.T, src string) (env []string, target string) { target = t.TempDir() } - testGoCoverage := TrackTestCoverage(t) - if testGoCoverage == "" { - return []string{}, target - } - - coverDir := filepath.Dir(testGoCoverage) - if c := os.Getenv("RAW_COVER_DIR"); c != "" { - coverDir = c + coverDir := filepath.Join(CoverDir(), "rust-cov") + if coverDir == "" { + return nil, target } t.Cleanup(func() { - rustJSONCoverage := testGoCoverage + ".json" + rustJSONCoverage := filepath.Join(coverDir, "rust_coverage.json") //nolint:gosec // G204 we define what we cover ourself cmd := exec.Command("grcov", coverDir, "--binary-path", filepath.Join(target, "debug"), @@ -113,7 +96,7 @@ func TrackRustCoverage(t *testing.T, src string) (env []string, target string) { require.NoError(t, err, "Teardown: decode our json coverage file") // This is the destination file for rust coverage in go format. - outF, err := os.Create(testGoCoverage) + outF, err := os.Create(filepath.Join(coverDir, "rust2go_coverage")) require.NoErrorf(t, err, "Teardown: failed opening output golang compatible cover file: %s", err) defer func() { assert.NoError(t, outF.Close(), "Teardown: can’t close golang compatible cover file") }() @@ -189,15 +172,3 @@ func convertRustFileResult(t *testing.T, results []interface{}, p string, w io.W writeGoCoverageLine(t, w, p, l+1, 9999, covered) } } - -// markForTestCache list all root directories/files so that they are marked in the test cache. -func markForTestCache(t *testing.T, roots []string) { - t.Helper() - - for _, root := range roots { - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - return nil - }) - require.NoError(t, err, "Setup: Error when listing input files for caching handling") - } -} diff --git a/nss/integration-tests/helper_test.go b/nss/integration-tests/helper_test.go index 8ee964476..f80e64ef5 100644 --- a/nss/integration-tests/helper_test.go +++ b/nss/integration-tests/helper_test.go @@ -19,7 +19,7 @@ import ( "github.com/ubuntu/authd/internal/testutils" ) -var daemonPath, rawCovDir string +var daemonPath string // buildRustNSSLib builds the NSS library and links the compiled file to libPath. func buildRustNSSLib(t *testing.T) { @@ -32,9 +32,8 @@ func buildRustNSSLib(t *testing.T) { cargo = "cargo" } - rustDir := filepath.Join(projectRoot, "nss") - testutils.MarkRustFilesForTestCache(t, rustDir) var target string + rustDir := filepath.Join(projectRoot, "nss") rustCovEnv, target = testutils.TrackRustCoverage(t, rustDir) // Builds the nss library. @@ -115,12 +114,7 @@ paths: // #nosec:G204 - we control the command arguments in tests cmd := exec.Command(daemonPath, "-c", configPath) cmd.Stderr = os.Stderr - if testutils.WantCoverage() { - if rawCovDir == "" { - t.Fatalf("Setup: coverage is wanted but no coverage dir was set") - } - cmd.Env = append(cmd.Env, fmt.Sprintf("GOCOVERDIR=%s", rawCovDir)) - } + cmd.Env = testutils.AppendCovEnv(cmd.Env) stopped = make(chan struct{}) go func() { @@ -152,7 +146,7 @@ func buildDaemon() (execPath string, cleanup func(), err error) { execPath = filepath.Join(tempDir, "authd") cmd := exec.Command("go", "build") cmd.Dir = projectRoot - if testutils.WantCoverage() { + if testutils.CoverDir() != "" { // -cover is a "positional flag", so it needs to come right after the "build" command. cmd.Args = append(cmd.Args, "-cover") } diff --git a/nss/integration-tests/nss_test.go b/nss/integration-tests/nss_test.go index 0f024797d..d8805ee74 100644 --- a/nss/integration-tests/nss_test.go +++ b/nss/integration-tests/nss_test.go @@ -3,11 +3,9 @@ package nss_test import ( "context" "flag" - "fmt" "log" "os" "os/exec" - "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -134,44 +132,5 @@ func TestMain(m *testing.M) { defer cleanup() daemonPath = execPath - // Creates the directory to store the coverage files for the integration tests. - if testutils.WantCoverage() { - rawCovDir = os.Getenv("RAW_COVER_DIR") - if rawCovDir == "" { - dir, err := os.MkdirTemp("", "authd-coverage") - if err != nil { - log.Printf("Setup: failed to create temp dir for coverage: %v", err) - cleanup() - os.Exit(24) - } - defer os.RemoveAll(dir) - rawCovDir = dir - } - } - - code := m.Run() - if code == 0 && rawCovDir != "" { - coverprofile := filepath.Join(rawCovDir, "integration-tests.coverprofile") - // #nosec:G204 - we control the command arguments in tests - cmd := exec.Command("go", "tool", "covdata", "textfmt", fmt.Sprintf("-i=%s", rawCovDir), fmt.Sprintf("-o=%s", coverprofile)) - if err := cmd.Run(); err != nil { - log.Printf("Teardown: failed to parse coverage files: %v", err) - cleanup() - os.RemoveAll(rawCovDir) - os.Exit(24) - } - testutils.MarkCoverageForMerging(coverprofile) - } - - if err := testutils.MergeCoverages(); err != nil { - log.Printf("Teardown: failed to merge coverage files: %v", err) - - // This ensures that we fail the test if we can't merge the coverage files, if the test - // was successful, otherwise we exit with the code returned by m.Run() - if code == 0 { - cleanup() - os.RemoveAll(rawCovDir) - defer os.Exit(24) - } - } + m.Run() }