handle run count within testutil

Signed-off-by: CrazyMax <[email protected]>

.github/workflows/.test.yml

@@ -108,6 +108,7 @@ jobs:
       TESTFLAGS: -v --timeout=30m
       TEST_IMAGE_BUILD: 0
       TEST_IMAGE_ID: buildkit-bench
+      BUILDKIT_RUN_COUNT: 3
     steps:
       -
         name: Download config files

hack/test

@@ -7,6 +7,9 @@ set -eu -o pipefail
 : "${TEST_KEEP_CACHE=}"
 : "${TESTFLAGS=}"
 
+: "${BUILDKIT_REF_RANDOM=}"
+: "${BUILDKIT_RUN_COUNT=1}"
+
 gotestArgs="-mod=vendor"
 
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
@@ -26,13 +29,8 @@ set -x
 docker run --rm --privileged \
   -v /tmp \
   --volumes-from=$cacheVolume \
+  -e BUILDKIT_REF_RANDOM \
+  -e BUILDKIT_RUN_COUNT \
   -e REGISTRY_MIRROR_DIR=/root/.cache/registry \
   $TEST_IMAGE_ID \
-  go test $gotestArgs ${TESTFLAGS:--v} ${TESTPKGS:-./...}
-
-docker run --rm --privileged \
-  -v /tmp \
-  --volumes-from=$cacheVolume \
-  -e REGISTRY_MIRROR_DIR=/root/.cache/registry \
-  $TEST_IMAGE_ID \
-  go test -bench . -benchmem -count=3 -run ^$ $gotestArgs ${TESTFLAGS:--v} ${TESTPKGS:-./...}
+  go test -bench . -benchmem $gotestArgs ${TESTFLAGS:--v} ${TESTPKGS:-./...}

util/testutil/run.go

@@ -8,6 +8,7 @@ import (
 	"reflect"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -152,26 +153,39 @@ func Run(tb testing.TB, runners []Runner, opt ...TestOpt) {
 		list = []Worker{list[rng.Intn(len(list))]}
 	}
 
+	runCount := 1
+	if v, ok := os.LookupEnv("BUILDKIT_RUN_COUNT"); ok && v != "" {
+		if _, ok := tb.(*testing.B); ok {
+			if count, err := strconv.Atoi(v); err != nil {
+				tb.Fatalf("invalid BUILDKIT_RUN_COUNT value: %v", err)
+			} else {
+				runCount = count
+			}
+		}
+	}
+
 	for _, br := range list {
 		for _, runner := range runners {
 			for _, mv := range matrix {
-				fn := runner.Name()
-				name := fn + "/ref=" + br.Name() + mv.functionSuffix()
-				func(fn, testName string, br Worker, runner Runner, mv matrixValue) {
-					ok := runTest(tb, testName, func(tb testing.TB) {
-						ctx := appcontext.Context()
-						require.NoError(tb, sandboxLimiter.Acquire(context.TODO(), 1))
-						defer sandboxLimiter.Release(1)
-
-						ctx, cancel := context.WithCancelCause(ctx)
-						defer cancel(errors.WithStack(context.Canceled))
-
-						sb, _, err := newSandbox(ctx, br, mv)
-						require.NoError(tb, err)
-						runner.Run(tb, sb)
-					})
-					require.True(tb, ok)
-				}(fn, name, br, runner, mv)
+				for rc := 1; rc <= runCount; rc++ {
+					fn := runner.Name()
+					name := fn + "/ref=" + br.Name() + mv.functionSuffix() + "/run=" + strconv.Itoa(rc)
+					func(fn, testName string, br Worker, runner Runner, mv matrixValue) {
+						ok := runTest(tb, testName, func(tb testing.TB) {
+							ctx := appcontext.Context()
+							require.NoError(tb, sandboxLimiter.Acquire(context.TODO(), 1))
+							defer sandboxLimiter.Release(1)
+
+							ctx, cancel := context.WithCancelCause(ctx)
+							defer cancel(errors.WithStack(context.Canceled))
+
+							sb, _, err := newSandbox(ctx, br, mv)
+							require.NoError(tb, err)
+							runner.Run(tb, sb)
+						})
+						require.True(tb, ok)
+					}(fn, name, br, runner, mv)
+				}
 			}
 		}
 	}