diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 16e87a00dd97..1060e3cc4a8f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -275,6 +275,14 @@ jobs: annotate_only: true skip_annotations: true + - name: Generate code coverage + run: make ./.testoutput/summary.cover.out + - name: Upload code coverage + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + file: ./.testoutput/summary.cover.out + - name: Upload test results # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. uses: actions/upload-artifact@v4.4.3 @@ -352,7 +360,15 @@ jobs: detailed_summary: true check_annotations: false annotate_only: true - skip_annotations: true + skip_annotations: tru + + - name: Generate code coverage + run: make ./.testoutput/summary.cover.out + - name: Upload code coverage + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + file: ./.testoutput/summary.cover.out - name: Upload test results # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. @@ -502,7 +518,15 @@ jobs: detailed_summary: true check_annotations: false annotate_only: true - skip_annotations: true + skip_annotations: tru + + - name: Generate code coverage + run: make ./.testoutput/summary.cover.out + - name: Upload code coverage + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + file: ./.testoutput/summary.cover.out - name: Upload test results # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. @@ -628,6 +652,14 @@ jobs: annotate_only: true skip_annotations: true + - name: Generate code coverage + run: make ./.testoutput/summary.cover.out + - name: Upload code coverage + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + file: ./.testoutput/summary.cover.out + - name: Upload test results # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. uses: actions/upload-artifact@v4.4.3 @@ -743,6 +775,14 @@ jobs: timeout-minutes: 15 run: make functional-test-ndc-coverage + - name: Generate code coverage + run: make ./.testoutput/summary.cover.out + - name: Upload code coverage + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + file: ./.testoutput/summary.cover.out + - name: Upload test results # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. uses: actions/upload-artifact@v4.4.3 diff --git a/Makefile b/Makefile index 4afd22a3ce78..0339d3f1a686 100644 --- a/Makefile +++ b/Makefile @@ -123,19 +123,21 @@ TEST_OUTPUT_ROOT := ./.testoutput NEW_COVER_PROFILE = $(TEST_OUTPUT_ROOT)/$(shell xxd -p -l 16 /dev/urandom).cover.out # generates a new filename each time it's substituted SUMMARY_COVER_PROFILE := $(TEST_OUTPUT_ROOT)/summary.cover.out NEW_REPORT = $(TEST_OUTPUT_ROOT)/$(shell xxd -p -l 16 /dev/urandom).junit.xml # generates a new filename each time it's substituted +COVERPKG = $(shell go list ./... | \ + grep -v /server/api/ | \ + grep -v /server/cmd/ | \ + grep -v /server/tests/ | \ + grep -v /server/tools/ | \ + grep -v /server/commong/testing/) # exclude generated packages entirely +COVERPKG_FLAG = -coverpkg="$(COVERPKG)" # indidivually generated files are removed when generating summary + +todo: + echo $(COVERPKG) # DB SQL_USER ?= temporal SQL_PASSWORD ?= temporal -# Need the following option to have integration and functional tests count towards coverage. godoc below: -# -coverpkg pkg1,pkg2,pkg3 -# Apply coverage analysis in each test to the given list of packages. -# The default is for each test to analyze only the package being tested. -# Packages are specified as import paths. -INTEGRATION_TEST_COVERPKG := -coverpkg="$(MODULE_ROOT)/common/persistence/..." -FUNCTIONAL_TEST_COVERPKG := -coverpkg="$(MODULE_ROOT)/client/...,$(MODULE_ROOT)/common/...,$(MODULE_ROOT)/service/...,$(MODULE_ROOT)/temporal/..." - # Only prints output if the exit code is non-zero define silent_exec @output=$$($(1) 2>&1); \ @@ -415,37 +417,37 @@ unit-test-coverage: prepare-coverage-test @printf $(COLOR) "Run unit tests with coverage..." go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(UNIT_TEST_DIRS) -- \ $(COMPILED_TEST_ARGS) \ - -coverprofile=$(NEW_COVER_PROFILE) + -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) integration-test-coverage: prepare-coverage-test @printf $(COLOR) "Run integration tests with coverage..." go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(INTEGRATION_TEST_DIRS) -- \ $(COMPILED_TEST_ARGS) \ - -coverprofile=$(NEW_COVER_PROFILE) $(INTEGRATION_TEST_COVERPKG) + -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) # This should use the same build flags as functional-test-coverage and functional-test-{xdc,ndc}-coverage for best build caching. pre-build-functional-test-coverage: prepare-coverage-test - go test -c -o /dev/null $(FUNCTIONAL_TEST_ROOT) $(TEST_ARGS) $(TEST_TAG_FLAG) $(FUNCTIONAL_TEST_COVERPKG) + go test -c -o /dev/null $(FUNCTIONAL_TEST_ROOT) $(TEST_ARGS) $(TEST_TAG_FLAG) $(COVERPKG_FLAG) functional-test-coverage: prepare-coverage-test @printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..." go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_ROOT) -- \ $(COMPILED_TEST_ARGS) \ - -coverprofile=$(NEW_COVER_PROFILE) $(FUNCTIONAL_TEST_COVERPKG) \ + -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) functional-test-xdc-coverage: prepare-coverage-test @printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..." go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_XDC_ROOT) -- \ $(COMPILED_TEST_ARGS) \ - -coverprofile=$(NEW_COVER_PROFILE) $(FUNCTIONAL_TEST_COVERPKG) \ + -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) functional-test-ndc-coverage: prepare-coverage-test @printf $(COLOR) "Run functional test for NDC with coverage with $(PERSISTENCE_DRIVER) driver..." go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_NDC_ROOT) -- \ $(COMPILED_TEST_ARGS) \ - -coverprofile=$(NEW_COVER_PROFILE) $(FUNCTIONAL_TEST_COVERPKG) \ + -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) .PHONY: $(SUMMARY_COVER_PROFILE) @@ -453,12 +455,16 @@ $(SUMMARY_COVER_PROFILE): @printf $(COLOR) "Combine coverage reports to $(SUMMARY_COVER_PROFILE)..." @rm -f $(SUMMARY_COVER_PROFILE) $(SUMMARY_COVER_PROFILE).html @if [ -z "$(wildcard $(TEST_OUTPUT_ROOT)/*.cover.out)" ]; then \ - echo "No coverage data, aborting!" && exit 1; \ + echo "No coverage data, skipping!"; \ fi @echo "mode: atomic" > $(SUMMARY_COVER_PROFILE) $(foreach COVER_PROFILE,$(wildcard $(TEST_OUTPUT_ROOT)/*.cover.out),\ @printf "Add %s...\n" $(COVER_PROFILE); \ - @grep -v -e "[Mm]ocks\?.go" -e "^mode: \w\+" $(COVER_PROFILE) >> $(SUMMARY_COVER_PROFILE) || true \ + grep -v \ + -e "^mode: \w\+" \ + -e "_gen.go" \ + -e "[Mm]ocks\?.go" \ + $(COVER_PROFILE) >> $(SUMMARY_COVER_PROFILE) || true \ $(NEWLINE)) coverage-report: $(SUMMARY_COVER_PROFILE) diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go index 99f0e9efdc7d..04e9a0e9885e 100644 --- a/tools/testrunner/testrunner.go +++ b/tools/testrunner/testrunner.go @@ -39,15 +39,21 @@ import ( "github.com/google/uuid" ) +const ( + codeCoverageExtension = ".cover.out" +) + type attempt struct { - runner *runner - number int - exitErr *exec.ExitError - junitReport *junitReport + runner *runner + number int + exitErr *exec.ExitError + coverprofilePath string + junitReport *junitReport } func (a *attempt) run(ctx context.Context, args []string) (string, error) { args = append([]string{"--junitfile", a.junitReport.path}, args...) + args = append([]string{"-coverprofile", a.coverprofilePath}, args...) log.Printf("starting test attempt %d with args: %v", a.number, args) cmd := exec.CommandContext(ctx, a.runner.gotestsumExecutable, args...) var output strings.Builder @@ -63,6 +69,7 @@ func (a *attempt) run(ctx context.Context, args []string) (string, error) { type runner struct { gotestsumExecutable string junitOutputPath string + coverprofilePath string attempts []*attempt retries int } @@ -81,7 +88,7 @@ func (r *runner) sanitizeAndParseArgs(args []string) ([]string, error) { err string } var next *action - for i, arg := range args { + for _, arg := range args { if next != nil { if err := next.f(arg); err != nil { return nil, err @@ -105,6 +112,8 @@ func (r *runner) sanitizeAndParseArgs(args []string) ([]string, error) { return nil, err } continue + } else if strings.HasPrefix(arg, "-coverprofile=") { + r.coverprofilePath = strings.Split(arg, "=")[1] } else if arg == "--junitfile" { // --junitfile is used by gotestsum next = &action{ @@ -119,16 +128,16 @@ func (r *runner) sanitizeAndParseArgs(args []string) ([]string, error) { // --junitfile is used by gotestsum r.junitOutputPath = arg[len("--junitfile="):] continue - } else if arg == "--" { - // Forward all arguments from -- on. - sanitizedArgs = append(sanitizedArgs, args[i:]...) - break } + // Pass through all other arguments. sanitizedArgs = append(sanitizedArgs, arg) } if next != nil { return nil, fmt.Errorf("incomplete command line arguments: %s", next.err) } + if r.coverprofilePath == "" { + return nil, fmt.Errorf("missing required argument -coverprofile") + } if r.junitOutputPath == "" { return nil, fmt.Errorf("missing required argument --junitfile") } @@ -139,6 +148,11 @@ func (r *runner) newAttempt() *attempt { a := &attempt{ runner: r, number: len(r.attempts) + 1, + coverprofilePath: fmt.Sprintf( + "%v_%v%v", + strings.TrimSuffix(r.coverprofilePath, codeCoverageExtension), + len(r.attempts), + codeCoverageExtension), junitReport: &junitReport{ path: filepath.Join(os.TempDir(), fmt.Sprintf("temporalio-temporal-%s-junit.xml", uuid.NewString())), }, diff --git a/tools/testrunner/testrunner_test.go b/tools/testrunner/testrunner_test.go index 466be18dd34f..40dcfd38cee4 100644 --- a/tools/testrunner/testrunner_test.go +++ b/tools/testrunner/testrunner_test.go @@ -67,7 +67,7 @@ func TestRunnerSanitizeAndParseArgs(t *testing.T) { _, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "-retries=invalid"}) require.ErrorContains(t, err, `strconv.Atoi: parsing "invalid"`) }) - t.Run("JuintfileSingleArg", func(t *testing.T) { + t.Run("JunitfileSingleArg", func(t *testing.T) { r := newRunner("gotestsum") args, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "--junitfile=foo"}) require.NoError(t, err) @@ -86,6 +86,13 @@ func TestRunnerSanitizeAndParseArgs(t *testing.T) { require.Equal(t, []string{"-foo", "bar", "--", "-retries=3"}, args) require.Equal(t, 0, r.retries) }) + t.Run("Coverprofile", func(t *testing.T) { + r := newRunner("gotestsum") + args, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "--junitfile=foo", "--", "-coverprofile=foo.cover.out"}) + require.NoError(t, err) + r.coverprofilePath = "foo.cover.out" + require.Equal(t, []string{"-foo", "bar", "--", "-coverprofile=foo.cover.out"}, args) + }) } func TestStripRunFromArgs(t *testing.T) {