Merge cockroachdb#125707

125707: roachtest: ignore workload for 5 mins after start in wal failover r=RaduBerinde a=itsbilal Previously, we'd look at p99 latencies for the workload since its very start, in the disk-stall/wal-failover roachtest. This was relatively ambitious as the workload is a high-concurrency kv workload with no ramping period at the start, so the chance of high p99 latency even under normal performance is high. This change ignores the workload's metrics from the first 5 mins of the workload (as opposed to just the first minute), and explicitly adds a 1min ramp period to the workload where concurrency is gradually increased. Fixes cockroachdb#124977. Epic: none Release note: None Co-authored-by: Bilal Akhtar <[email protected]>
nameisbhaskar · Jun 15, 2024 · 3aa5bdf · 3aa5bdf
2 parents 9a86e0b + bc1504a
commit 3aa5bdf
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/pkg/cmd/roachtest/tests/disk_stall.go b/pkg/cmd/roachtest/tests/disk_stall.go
@@ -103,7 +103,7 @@ func runDiskStalledWALFailover(
 	m := c.NewMonitor(ctx, c.Range(1, 3))
 	m.Go(func(ctx context.Context) error {
 		c.Run(ctx, option.WithNodes(c.Node(4)), `./cockroach workload run kv --read-percent 0 `+
-			`--duration 60m --concurrency 4096 --max-rate 4096 --tolerate-errors `+
+			`--duration 60m --concurrency 4096 --ramp=1m --max-rate 4096 --tolerate-errors `+
 			` --min-block-bytes=2048 --max-block-bytes=2048 --timeout 1s `+
 			`{pgurl:1-3}`)
 		return nil
@@ -160,7 +160,7 @@ func runDiskStalledWALFailover(
 	}
 
 	data := mustGetMetrics(ctx, c, t, adminURL,
-		workloadStartAt.Add(time.Minute),
+		workloadStartAt.Add(5*time.Minute),
 		timeutil.Now().Add(-time.Minute),
 		[]tsQuery{
 			{name: "cr.node.sql.exec.latency-p99.99", queryType: total, sources: []string{"2"}},