From d52859e6293b2c073458e5d02a0bc1dab8251d5a Mon Sep 17 00:00:00 2001
From: Matthew Sackman <matthew@cue.works>
Date: Tue, 20 Aug 2024 14:00:52 +0100
Subject: [PATCH] cue/benchmarks: move benchmark code out of testdata

bench_test.go used to reside in cue/testdata/benchmarks

This meant that a `go test bench=. ./...` would not run these
benchmarks, because go test never delves inside testdata directories.

- Move bench_test to the cue directory (with package cue_test)
- Make it use filepath.WalkDir (matching cuetxtar)
- Make it use the eval matrices so that the benchmarks get run against
  both the old and new evaluators

Signed-off-by: Matthew Sackman <matthew@cue.works>
Change-Id: I85fb895579ad38039eb47176eeb48c1aaceda9ea
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1199739
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Marcel van Lohuizen <mpvl@gmail.com>
---
 cue/{testdata/benchmarks => }/bench_test.go | 56 ++++++++++++---------
 1 file changed, 32 insertions(+), 24 deletions(-)
 rename cue/{testdata/benchmarks => }/bench_test.go (68%)

diff --git a/cue/testdata/benchmarks/bench_test.go b/cue/bench_test.go
similarity index 68%
rename from cue/testdata/benchmarks/bench_test.go
rename to cue/bench_test.go
index 2e2417a4758..fe9ef8089ca 100644
--- a/cue/testdata/benchmarks/bench_test.go
+++ b/cue/bench_test.go
@@ -12,45 +12,48 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package benchmarks
+package cue_test
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
 
-	"cuelang.org/go/cue"
 	"cuelang.org/go/internal/core/eval"
 	"cuelang.org/go/internal/core/runtime"
+	"cuelang.org/go/internal/cuetdtest"
 	"cuelang.org/go/internal/cuetest"
 	"cuelang.org/go/internal/cuetxtar"
 	"golang.org/x/tools/txtar"
 )
 
+var (
+	matrix = cuetdtest.FullMatrix
+)
+
 func Benchmark(b *testing.B) {
-	entries, err := os.ReadDir(".")
-	if err != nil {
-		b.Fatal(err)
-	}
+	root := "testdata/benchmarks"
+	err := filepath.WalkDir(root, func(fullpath string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
 
-	for _, entry := range entries {
-		name := entry.Name()
-		if entry.IsDir() || filepath.Ext(name) != ".txtar" {
-			continue
+		if entry.IsDir() || filepath.Ext(fullpath) != ".txtar" {
+			return nil
 		}
 
-		a, err := txtar.ParseFile(name)
+		a, err := txtar.ParseFile(fullpath)
 		if err != nil {
-			b.Fatal(err)
+			return err
 		}
 
 		inst := cuetxtar.Load(a, b.TempDir())[0]
 		if inst.Err != nil {
-			b.Fatal(inst.Err)
+			return inst.Err
 		}
 
 		r := runtime.New()
-
 		v, err := r.Build(nil, inst)
 		if err != nil {
 			b.Fatal(err)
@@ -83,19 +86,24 @@ func Benchmark(b *testing.B) {
 			if err != nil {
 				b.Fatal(err)
 			}
-			os.WriteFile(name, txtar.Format(a), info.Mode())
+			os.WriteFile(fullpath, txtar.Format(a), info.Mode())
 		}
 
-		b.Run(name, func(b *testing.B) {
-			b.ReportAllocs()
-			for i := 0; i < b.N; i++ {
-				inst := cue.Build(cuetxtar.Load(a, b.TempDir()))[0]
-				if inst.Err != nil {
-					b.Fatal(inst.Err)
-				}
-
-				inst.Value().Validate()
+		b.Run(entry.Name(), func(b *testing.B) {
+			for _, m := range matrix {
+				b.Run(m.Name(), func(b *testing.B) {
+					b.ReportAllocs()
+					for i := 0; i < b.N; i++ {
+						ctx := m.Context()
+						value := ctx.BuildInstance(cuetxtar.Load(a, b.TempDir())[0])
+						value.Validate()
+					}
+				})
 			}
 		})
+		return nil
+	})
+	if err != nil {
+		b.Fatal(err)
 	}
 }