ci: report coverage in Cobertura XML format (#946)

The simplest way to report XML coverage in our multi-language project is
to separate coverage generation per language, using bespoke tools to
generate/convert to XML (we use `python3-coverage` for Python, `gocov`
to convert Go coverage to an intermediate JSON format, and `gocov-xml`
to convert the JSON to Cobertura XML). Additionally, `reportgenerator`
is used to combine the resulting XML files into a single report, which
is then uploaded to Codecov.

The way it works under the hood is:
- skip Python XML generation if `reportgenerator`, `gocov`, or
`gocov-xml` do not exist in the `PATH`
- if the executables are present, Python coverage is converted to XML
and copied to `$PROJECTROOT/coverage`
- as part of the QA action, the Go-only coverage is converted to XML
- the Python and Go XML files are combined to a single report using
`reportgenerator`

It's important to note that existing coverage generation is not affected
by these changes, thus everything here is additive.

Fixes UDENG-2034

.github/workflows/qa.yaml

@@ -50,6 +50,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${{ env.apt_dependencies }}
+
+          # Coverage dependencies
+          go install github.com/AlekSi/gocov-xml@latest
+          go install github.com/axw/gocov/gocov@latest
+          dotnet tool install -g dotnet-reportgenerator-globaltool
       - name: Set required environment variables
         run: echo "SUDO_PACKAGES=$(cat debian/tests/.sudo-packages)" >> $GITHUB_ENV
       - name: Authenticate to docker local registry and pull image with our token
@@ -69,9 +74,23 @@ jobs:
           sudo -E $(which go) test -coverpkg=./... -coverprofile=/tmp/coverage.sudo.out -covermode=set $SUDO_PACKAGES
 
           # Combine coverage files, and filter out test utilities and generated files
-          cod_cov_dir="$(pwd)/coverage/codecov"
+          coverage_dir="$(pwd)/coverage"
+          cod_cov_dir="${coverage_dir}/codecov"
           mkdir -p "${cod_cov_dir}"
-          grep -hv -e "testutils" -e "pb.go:" -e "/e2e/" /tmp/coverage.out /tmp/coverage.sudo.out > "${cod_cov_dir}/coverage.combined.out"
+          combined_cov_file="${coverage_dir}/coverage.combined.out"
+          go_only_cov_file="${coverage_dir}/coverage.go-only.out"
+          echo "mode: set" > "${combined_cov_file}"
+          grep -hv -e "testutils" -e "pb.go:" -e "/e2e/" -e "mode: set" /tmp/coverage.out /tmp/coverage.sudo.out >> "${combined_cov_file}"
+
+          # Prepare XML coverage report
+          grep -hv -e "adsys-gpolist" -e "cert-autoenroll" "${combined_cov_file}" > "${go_only_cov_file}"
+          gocov convert "${go_only_cov_file}" | gocov-xml > "${coverage_dir}/coverage.xml"
+          reportgenerator -reports:"${coverage_dir}/*.xml" -targetdir:"${cod_cov_dir}" -reporttypes:Cobertura
+      - name: Upload XML coverage report as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage.xml
+          path: ./coverage/codecov/Cobertura.xml
       - name: Run tests (with race detector)
         run: |
           go test -race ./...
@@ -80,7 +99,7 @@ jobs:
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
-          directory: ./coverage/codecov
+          file: ./coverage/codecov/Cobertura.xml
           token: ${{ secrets.CODECOV_TOKEN }}
 
   adwatchd-tests:

internal/testutils/coverage.go

@@ -6,14 +6,15 @@ import (
 	"io"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"strings"
 	"sync"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/termie/go-shutil"
 )
 
 var (
@@ -22,16 +23,55 @@ var (
 
 	coveragesToMerge   []string
 	coveragesToMergeMu sync.Mutex
+
+	generateXMLCoverage bool
+)
+
+const (
+	goCoverage  = "go"  // Go coverage format
+	xmlCoverage = "xml" // XML (Cobertura) coverage format
 )
 
+type coverageOptions struct {
+	coverageFormat string
+}
+
+// CoverageOption represents an optional function that can be used to override
+// some of the coverage default values.
+type CoverageOption func(*coverageOptions)
+
+// WithCoverageFormat overrides the default coverage format, impacting the
+// filename of the coverage file.
+func WithCoverageFormat(coverageFormat string) CoverageOption {
+	return func(o *coverageOptions) {
+		if coverageFormat != "" {
+			o.coverageFormat = coverageFormat
+		}
+	}
+}
+
+func init() {
+	// XML coverage generation is a best effort, so we don't fail if the required tools are not found.
+	if commandExists("reportgenerator") && commandExists("gocov") && commandExists("gocov-xml") {
+		generateXMLCoverage = true
+	}
+}
+
 // TrackTestCoverage starts tracking coverage in a dedicated file based on current test name.
 // This file will be merged to the current coverage main file.
 // It’s up to the test use the returned path to file golang-compatible cover format content.
 // To collect all coverages, then MergeCoverages() should be called after m.Run().
 // If coverage is not enabled, nothing is done.
-func TrackTestCoverage(t *testing.T) (testCoverFile string) {
+func TrackTestCoverage(t *testing.T, opts ...CoverageOption) (testCoverFile string) {
 	t.Helper()
 
+	args := coverageOptions{
+		coverageFormat: goCoverage,
+	}
+	for _, o := range opts {
+		o(&args)
+	}
+
 	goMainCoverProfileOnce.Do(func() {
 		for _, arg := range os.Args {
 			if !strings.HasPrefix(arg, "-test.coverprofile=") {
@@ -48,9 +88,11 @@ func TrackTestCoverage(t *testing.T) (testCoverFile string) {
 	coverAbsPath, err := filepath.Abs(goMainCoverProfile)
 	require.NoError(t, err, "Setup: can't transform go cover profile to absolute path")
 
-	testCoverFile = fmt.Sprintf("%s.%s", coverAbsPath, strings.ReplaceAll(
-		strings.ReplaceAll(t.Name(), "/", "_"),
-		"\\", "_"))
+	testCoverFile = fmt.Sprintf("%s.%s.%s",
+		coverAbsPath,
+		strings.ReplaceAll(strings.ReplaceAll(t.Name(), "/", "_"), "\\", "_"),
+		args.coverageFormat,
+	)
 	coveragesToMergeMu.Lock()
 	defer coveragesToMergeMu.Unlock()
 	if slices.Contains(coveragesToMerge, testCoverFile) {
@@ -67,7 +109,27 @@ func TrackTestCoverage(t *testing.T) (testCoverFile string) {
 func MergeCoverages() {
 	coveragesToMergeMu.Lock()
 	defer coveragesToMergeMu.Unlock()
+
+	projectRoot, err := projectRoot(".")
+	if err != nil {
+		log.Fatalf("Teardown: can't find project root: %v", err)
+	}
+
+	if err := os.MkdirAll(filepath.Join(projectRoot, "coverage"), 0700); err != nil {
+		log.Fatalf("Teardown: can’t create coverage directory: %v", err)
+	}
+
+	// Merge Go coverage files
 	for _, cov := range coveragesToMerge {
+		// For XML coverage files, we just copy them to a persistent directory
+		// for future manipulation.
+		if strings.HasSuffix(cov, "."+xmlCoverage) {
+			if err := shutil.CopyFile(cov, filepath.Join(projectRoot, "coverage", filepath.Base(cov)), false); err != nil {
+				log.Fatalf("Teardown: can’t copy coverage file to project root: %v", err)
+			}
+			continue
+		}
+
 		if err := appendToFile(cov, goMainCoverProfile); err != nil {
 			log.Fatalf("Teardown: can’t inject coverage into the golang one: %v", err)
 		}
@@ -127,32 +189,44 @@ func appendToFile(src, dst string) error {
 func fqdnToPath(t *testing.T, path string) string {
 	t.Helper()
 
-	srcPath, err := filepath.Abs(path)
-	require.NoError(t, err, "Setup: can't calculate absolute path")
+	absPath, err := filepath.Abs(path)
+	require.NoError(t, err, "Setup: can't transform path to absolute path")
+
+	projectRoot, err := projectRoot(path)
+	require.NoError(t, err, "Setup: can't find project root")
+
+	f, err := os.Open(filepath.Join(projectRoot, "go.mod"))
+	require.NoError(t, err, "Setup: can't open go.mod")
+
+	r := bufio.NewReader(f)
+	l, err := r.ReadString('\n')
+	require.NoError(t, err, "can't read go.mod first line")
+	if !strings.HasPrefix(l, "module ") {
+		t.Fatal(`Setup: failed to find "module" line in go.mod`)
+	}
+
+	prefix := strings.TrimSpace(strings.TrimPrefix(l, "module "))
+	relpath := strings.TrimPrefix(absPath, projectRoot)
+	return filepath.Join(prefix, relpath)
+}
+
+// projectRoot returns the root of the project by looking for a go.mod file.
+func projectRoot(path string) (string, error) {
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return "", fmt.Errorf("can't calculate absolute path: %w", err)
+	}
 
-	d := srcPath
-	for d != "/" {
-		f, err := os.Open(filepath.Clean(filepath.Join(d, "go.mod")))
+	for absPath != "/" {
+		_, err := os.Stat(filepath.Clean(filepath.Join(absPath, "go.mod")))
 		if err != nil {
-			d = filepath.Dir(d)
+			absPath = filepath.Dir(absPath)
 			continue
 		}
-		defer func() { assert.NoError(t, f.Close(), "Setup: can’t close go.mod") }()
 
-		r := bufio.NewReader(f)
-		l, err := r.ReadString('\n')
-		require.NoError(t, err, "can't read go.mod first line")
-		if !strings.HasPrefix(l, "module ") {
-			t.Fatal(`Setup: failed to find "module" line in go.mod`)
-		}
-
-		prefix := strings.TrimSpace(strings.TrimPrefix(l, "module "))
-		relpath := strings.TrimPrefix(srcPath, d)
-		return filepath.Join(prefix, relpath)
+		return absPath, nil
 	}
-
-	t.Fatal("failed to find go.mod")
-	return ""
+	return "", fmt.Errorf("failed to find go.mod")
 }
 
 // writeGoCoverageLine writes given line in go coverage format to w.
@@ -162,3 +236,9 @@ func writeGoCoverageLine(t *testing.T, w io.Writer, file string, lineNum, lineLe
 	_, err := w.Write([]byte(fmt.Sprintf("%s:%d.1,%d.%d 1 %s\n", file, lineNum, lineNum, lineLength, covered)))
 	require.NoErrorf(t, err, "Teardown: can't write a write to golang compatible cover file : %v", err)
 }
+
+// commandExists returns true if the command exists in the PATH.
+func commandExists(cmd string) bool {
+	_, err := exec.LookPath(cmd)
+	return err == nil
+}

internal/testutils/pythoncoverage.go

@@ -30,6 +30,11 @@ func PythonCoverageToGoFormat(t *testing.T, include string, commandOnStdin bool)
 		return false
 	}
 
+	var testXMLCoverage string
+	if generateXMLCoverage {
+		testXMLCoverage = TrackTestCoverage(t, WithCoverageFormat(xmlCoverage))
+	}
+
 	// Check we have an executable "python3-coverage" in PATH for coverage request
 	_, err := exec.LookPath(coverageCmd)
 	require.NoErrorf(t, err, "Setup: coverage requested and no %s executable found in $PATH for python code", coverageCmd)
@@ -90,6 +95,13 @@ exec python3-coverage run -a %s $@
 		out, err := exec.Command(coverageCmd, "annotate", "-d", coverDir, "--include", tracedFile).CombinedOutput()
 		require.NoErrorf(t, err, "Teardown: can’t combine python coverage: %s", out)
 
+		// Generate XML report if supported
+		if testXMLCoverage != "" {
+			// #nosec G204 - we have a const for coverageCmd
+			out, err = exec.Command(coverageCmd, "xml", "-o", testXMLCoverage, "--include", tracedFile).CombinedOutput()
+			require.NoErrorf(t, err, "Teardown: can’t convert python coverage to XML: %s", out)
+		}
+
 		// Convert to golang compatible cover format
 		// The file will be transform with char_hexadecimal_filename_ext,cover if there is any / in the name.
 		// Matching it with global by filename.