From 3e3d0b17b56938d09514ca30059403ab916d2291 Mon Sep 17 00:00:00 2001 From: warber <72415058+warber@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:53:53 +0100 Subject: [PATCH] feat: TestProvider for easy, parallel-safe testing (#295) added experimental test-aware provider Signed-off-by: Bernd Warmuth Co-authored-by: Todd Baert Co-authored-by: Michael Beemer --- README.md | 74 +++++++--- go.sum | 20 --- openfeature/testing/testprovider.go | 122 +++++++++++++++++ openfeature/testing/testprovider_test.go | 167 +++++++++++++++++++++++ 4 files changed, 347 insertions(+), 36 deletions(-) create mode 100644 openfeature/testing/testprovider.go create mode 100644 openfeature/testing/testprovider_test.go diff --git a/README.md b/README.md index 6f450070..fb7785df 100644 --- a/README.md +++ b/README.md @@ -432,29 +432,71 @@ func (h MyHook) Error(context context.Context, hookContext openfeature.HookConte ## Testing -To test interactions with OpenFeature API and Client, you can rely on `openfeature.IEvaluation` & `openfeature.IClient` interfaces. +The SDK provides a `NewTestProvider` which allows you to set flags for the scope of a test. +The `TestProvider` is thread-safe and can be used in tests that run in parallel. -While you may use global methods to interact with the API, it is recommended to obtain the singleton API instance so that you can use appropriate mocks for your testing needs, +Call `testProvider.UsingFlags(t, tt.flags)` to set flags for a test, and clean them up with `testProvider.Cleanup()` ```go -// global helper -openfeature.SetProvider(myProvider) +import ( + "github.com/open-feature/go-sdk/openfeature" + "github.com/open-feature/go-sdk/openfeature/testing" +) -// singleton instance - preferred -apiInstance := openfeature.GetApiInstance() -apiInstance.SetProvider(myProvider) -``` +testProvider := NewTestProvider() +err := openfeature.GetApiInstance().SetProvider(testProvider) +if err != nil { + t.Errorf("unable to set provider") +} + +// configure flags for this test suite +tests := map[string]struct { + flags map[string]memprovider.InMemoryFlag + want bool +}{ + "test when flag is true": { + flags: map[string]memprovider.InMemoryFlag{ + "my_flag": { + State: memprovider.Enabled, + DefaultVariant: "on", + Variants: map[string]any{ + "on": true, + }, + }, + }, + want: true, + }, + "test when flag is false": { + flags: map[string]memprovider.InMemoryFlag{ + "my_flag": { + State: memprovider.Enabled, + DefaultVariant: "off", + Variants: map[string]any{ + "off": false, + }, + }, + }, + want: false, + }, +} -Similarly, while you have option (due to historical reasons) to create a client with `openfeature.NewClient()` helper, it is recommended to use API to generate the client which returns an `IClient` instance. +for name, tt := range tests { + tt := tt + name := name + t.Run(name, func(t *testing.T) { -```go -// global helper -openfeature.NewClient("myClient") + // be sure to clean up your flags + defer testProvider.Cleanup() + testProvider.UsingFlags(t, tt.flags) -// using API instance - preferred -apiInstance := openfeature.GetApiInstance() -apiInstance.GetClient() -apiInstance.GetNamedClient("myClient") + // your code under test + got := functionUnderTest() + + if got != tt.want { + t.Fatalf("uh oh, value is not as expected: got %v, want %v", got, tt.want) + } + }) +} ``` diff --git a/go.sum b/go.sum index 632b45c8..9a4b1728 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= -github.com/cucumber/godog v0.14.0 h1:h/K4t7XBxsFBF+UJEahNqJ1/2VHVepRXCSq3WWWnehs= -github.com/cucumber/godog v0.14.0/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= github.com/cucumber/godog v0.14.1 h1:HGZhcOyyfaKclHjJ+r/q93iaTJZLKYW6Tv3HkmUE6+M= github.com/cucumber/godog v0.14.1/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= github.com/cucumber/godog v0.15.0 h1:51AL8lBXF3f0cyA5CV4TnJFCTHpgiy+1x1Hb3TtZUmo= @@ -13,12 +11,9 @@ github.com/cucumber/messages/go/v22 v22.0.0/go.mod h1:aZipXTKc0JnjCsXrJnuZpWhtay github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -33,7 +28,6 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -59,12 +53,6 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= -golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -81,14 +69,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/openfeature/testing/testprovider.go b/openfeature/testing/testprovider.go new file mode 100644 index 00000000..fb42d3a3 --- /dev/null +++ b/openfeature/testing/testprovider.go @@ -0,0 +1,122 @@ +package testing + +import ( + "context" + "fmt" + "runtime" + "sync" + "testing" + + "github.com/open-feature/go-sdk/openfeature" + "github.com/open-feature/go-sdk/openfeature/memprovider" +) + +const testNameKey = "testName" + +// NewTestProvider creates a new `TestAwareProvider` +func NewTestProvider() TestProvider { + return TestProvider{ + providers: &sync.Map{}, + } +} + +// TestProvider is the recommended way to defined flags within the scope of a test. +// It uses the InMemoryProvider, with flags scoped per test. +// Before executing a test, specify the flag values to be used for the specific test using the UsingFlags function +type TestProvider struct { + openfeature.NoopProvider + providers *sync.Map +} + +// UsingFlags sets flags for the scope of a test +func (tp TestProvider) UsingFlags(test *testing.T, flags map[string]memprovider.InMemoryFlag) { + storeGoroutineLocal(test.Name()) + tp.providers.Store(test.Name(), memprovider.NewInMemoryProvider(flags)) +} + +// Cleanup deletes the flags provider bound to the current test and should be executed after each test execution +// e.g. using a defer statement. +func (tp TestProvider) Cleanup() { + tp.providers.Delete(getGoroutineLocal()) + deleteGoroutineLocal() +} + +func (tp TestProvider) BooleanEvaluation(ctx context.Context, flag string, defaultValue bool, flCtx openfeature.FlattenedContext) openfeature.BoolResolutionDetail { + return tp.getProvider().BooleanEvaluation(ctx, flag, defaultValue, flCtx) +} + +func (tp TestProvider) StringEvaluation(ctx context.Context, flag string, defaultValue string, flCtx openfeature.FlattenedContext) openfeature.StringResolutionDetail { + return tp.getProvider().StringEvaluation(ctx, flag, defaultValue, flCtx) +} + +func (tp TestProvider) FloatEvaluation(ctx context.Context, flag string, defaultValue float64, flCtx openfeature.FlattenedContext) openfeature.FloatResolutionDetail { + return tp.getProvider().FloatEvaluation(ctx, flag, defaultValue, flCtx) +} + +func (tp TestProvider) IntEvaluation(ctx context.Context, flag string, defaultValue int64, flCtx openfeature.FlattenedContext) openfeature.IntResolutionDetail { + return tp.getProvider().IntEvaluation(ctx, flag, defaultValue, flCtx) +} + +func (tp TestProvider) ObjectEvaluation(ctx context.Context, flag string, defaultValue interface{}, flCtx openfeature.FlattenedContext) openfeature.InterfaceResolutionDetail { + return tp.getProvider().ObjectEvaluation(ctx, flag, defaultValue, flCtx) +} + +func (tp TestProvider) Hooks() []openfeature.Hook { + return tp.NoopProvider.Hooks() +} + +func (tp TestProvider) Metadata() openfeature.Metadata { + return tp.NoopProvider.Metadata() +} + +func (tp TestProvider) getProvider() openfeature.FeatureProvider { + // Retrieve the test name from the goroutine-local storage. + testName, ok := getGoroutineLocal().(string) + if !ok { + panic("unable to detect test name; be sure to call `UsingFlags` in the scope of a test (in T.run)!") + } + + // Load the feature provider corresponding to the test name. + provider, ok := tp.providers.Load(testName) + if !ok { + panic("unable to find feature provider for given test name: " + testName) + } + + // Assert that the loaded provider is of type openfeature.FeatureProvider. + featureProvider, ok := provider.(openfeature.FeatureProvider) + if !ok { + panic("invalid type for feature provider for given test name: " + testName) + } + + return featureProvider +} + +var goroutineLocalData sync.Map + +func storeGoroutineLocal(value interface{}) { + gID := getGoroutineID() + goroutineLocalData.Store(fmt.Sprintf("%d_%v", gID, testNameKey), value) +} + +func getGoroutineLocal() interface{} { + gID := getGoroutineID() + value, _ := goroutineLocalData.Load(fmt.Sprintf("%d_%v", gID, testNameKey)) + return value +} + +func deleteGoroutineLocal() { + gID := getGoroutineID() + goroutineLocalData.Delete(fmt.Sprintf("%d_%v", gID, testNameKey)) +} + +func getGoroutineID() uint64 { + var buf [64]byte + n := runtime.Stack(buf[:], false) + stackLine := string(buf[:n]) + var gID uint64 + _, err := fmt.Sscanf(stackLine, "goroutine %d ", &gID) + if err != nil { + panic("unable to extract GID from stack trace") + } + return gID +} diff --git a/openfeature/testing/testprovider_test.go b/openfeature/testing/testprovider_test.go new file mode 100644 index 00000000..f2c2f07e --- /dev/null +++ b/openfeature/testing/testprovider_test.go @@ -0,0 +1,167 @@ +package testing + +import ( + "context" + "testing" + + "github.com/open-feature/go-sdk/openfeature" + "github.com/open-feature/go-sdk/openfeature/memprovider" +) + +func TestParallelSingletonUsage(t *testing.T) { + t.Parallel() + + testProvider := NewTestProvider() + err := openfeature.GetApiInstance().SetProvider(testProvider) + if err != nil { + t.Errorf("unable to set provider") + } + + tests := map[string]struct { + flags map[string]memprovider.InMemoryFlag + want bool + }{ + "test when flag is true": { + flags: map[string]memprovider.InMemoryFlag{ + "my_flag": { + State: memprovider.Enabled, + DefaultVariant: "on", + Variants: map[string]any{ + "on": true, + }, + }, + }, + want: true, + }, + "test when flag is false": { + flags: map[string]memprovider.InMemoryFlag{ + "my_flag": { + State: memprovider.Enabled, + DefaultVariant: "off", + Variants: map[string]any{ + "off": false, + }, + }, + }, + want: false, + }, + } + + for name, tt := range tests { + tt := tt + name := name + t.Run(name, func(t *testing.T) { + defer testProvider.Cleanup() + t.Parallel() + testProvider.UsingFlags(t, tt.flags) + + got := functionUnderTest() + + if got != tt.want { + t.Fatalf("uh oh, value is not as expected: got %v, want %v", got, tt.want) + } + }) + } +} + +func TestTestAwareProvider(t *testing.T) { + taw := NewTestProvider() + + flags := map[string]memprovider.InMemoryFlag{ + "ff-bool": { + State: memprovider.Enabled, + DefaultVariant: "variant_1", + Variants: map[string]any{ + "variant_1": true, + }, + }, + "ff-string": { + State: memprovider.Enabled, + DefaultVariant: "variant_1", + Variants: map[string]any{ + "variant_1": "str", + }, + }, + "ff-int": { + State: memprovider.Enabled, + DefaultVariant: "variant_1", + Variants: map[string]any{ + "variant_1": 1, + }, + }, + "ff-float": { + State: memprovider.Enabled, + DefaultVariant: "variant_1", + Variants: map[string]any{ + "variant_1": float64(1), + }, + }, + "ff-obj": { + State: memprovider.Enabled, + DefaultVariant: "variant_1", + Variants: map[string]any{ + "variant_1": "obj", + }, + }, + } + + t.Run("test bool evaluation", func(t *testing.T) { + taw.UsingFlags(t, flags) + result := taw.BooleanEvaluation(context.TODO(), "ff-bool", false, openfeature.FlattenedContext{}) + if result.Value != true { + t.Errorf("got %v, want %v", result, true) + } + }) + + t.Run("test string evaluation", func(t *testing.T) { + taw.UsingFlags(t, flags) + result := taw.StringEvaluation(context.TODO(), "ff-string", "otherStr", openfeature.FlattenedContext{}) + if result.Value != "str" { + t.Errorf("got %v, want %v", result, true) + } + }) + + t.Run("test int evaluation", func(t *testing.T) { + taw.UsingFlags(t, flags) + result := taw.IntEvaluation(context.TODO(), "ff-int", int64(2), openfeature.FlattenedContext{}) + if result.Value != 1 { + t.Errorf("got %v, want %v", result, true) + } + }) + + t.Run("test float evaluation", func(t *testing.T) { + taw.UsingFlags(t, flags) + result := taw.FloatEvaluation(context.TODO(), "ff-float", float64(2), openfeature.FlattenedContext{}) + if result.Value != float64(1) { + t.Errorf("got %v, want %v", result, true) + } + }) + t.Run("test obj evaluation", func(t *testing.T) { + taw.UsingFlags(t, flags) + result := taw.ObjectEvaluation(context.TODO(), "ff-obj", "stringobj", openfeature.FlattenedContext{}) + if result.Value != "obj" { + t.Errorf("got %v, want %v", result, true) + } + }) +} + +func Test_TestAwareProviderPanics(t *testing.T) { + + t.Run("provider panics if no test name was provided by calling SetProvider()", func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("the test aware provider did not panic") + } + }() + + taw := NewTestProvider() + taw.BooleanEvaluation(context.TODO(), "my-flag", true, openfeature.FlattenedContext{}) + }) +} + +func functionUnderTest() bool { + got := openfeature.GetApiInstance(). + GetClient(). + Boolean(context.TODO(), "my_flag", false, openfeature.EvaluationContext{}) + return got +}