From 81cb78ba051da86add66ca385fa7c540f340e117 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 7 Sep 2023 16:53:14 -0400 Subject: [PATCH] feat: flagd gherkin suite and flags (#54) Signed-off-by: Todd Baert --- .gherkin-lintrc | 4 +- README.md | 25 ++++++- flagd/Dockerfile | 12 +++- flags/custom-ops.json | 104 +++++++++++++++++++++++++++ flags/evaluator-refs.json | 52 ++++++++++++++ flags/testing-flags.json | 44 ------------ flags/zero-flags.json | 36 ++++++++++ gherkin/flagd-json-evaluator.feature | 63 ++++++++++++++++ gherkin/flagd.feature | 36 ++++++++++ sync/Dockerfile | 9 ++- 10 files changed, 334 insertions(+), 51 deletions(-) create mode 100644 flags/custom-ops.json create mode 100644 flags/evaluator-refs.json create mode 100644 flags/zero-flags.json create mode 100644 gherkin/flagd-json-evaluator.feature create mode 100644 gherkin/flagd.feature diff --git a/.gherkin-lintrc b/.gherkin-lintrc index abe95f9..16d3030 100644 --- a/.gherkin-lintrc +++ b/.gherkin-lintrc @@ -14,7 +14,9 @@ "Feature": 0, "Background": 2, "Scenario": 2, - "Step": 4 + "Step": 4, + "Examples": 4, + "example": 6 } ], "no-trailing-spaces": "on", diff --git a/README.md b/README.md index b846640..44ab2aa 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,27 @@ See the [flagd docs](https://flagd.dev/) for more information on flagd. The _sync_-testbed_ container is a docker image built on conforming to the sync.proto - a grpc which flagd or flagd in-process providers can use as a sync-source. It features an identical set of flags to the [flagd-testbed container](#flagd-testbed-container) -For details on the sync-testbed, see [sync/README.me](sync/README.md) \ No newline at end of file +For details on the sync-testbed, see [sync/README.me](sync/README.md) + +## Gherkin test suite + +The [gherkin/](gherkin/) dir includes a set of [_gherkin_](https://cucumber.io/docs/gherkin/) tests that define expected behavior associated with the configurations defined in the flagd-testbed (see [flags/](flags/)). +Combined with the _flagd-provider_ for the associated SDK and the flagd-testbed, these comprise a complete integration test suite. + +Included suites: + +[flagd.feature](gherkin/flagd.feature) includes tests relevant to flagd and all flagd providers: +* default value (zero-value) handling (some proto3 implementations handle these inconsistently). +* basic event handling + +[flagd-json-evaluator.feature](gherkin/flagd-json-evaluator.feature) includes tests relevant to flagd and in-process providers: +* custom JSONLogic operators (`starts_with`, `ends_with`, `fractional`, `sem_ver`) +* evaluator reuse via `$ref` + + +### Lint Gherkin files + +The Gherkin files structure can be linted using [gherkin-lint](https://github.com/vsiakka/gherkin-lint). The following commands require Node.js 10 or later. + +1. npm install +1. npm run gherkin-lint \ No newline at end of file diff --git a/flagd/Dockerfile b/flagd/Dockerfile index 101ad9e..caad665 100644 --- a/flagd/Dockerfile +++ b/flagd/Dockerfile @@ -1,10 +1,16 @@ -FROM ghcr.io/open-feature/flagd:v0.6.3 as flagd +# we NEED flagd v0.6.4 as a minimum +FROM ghcr.io/open-feature/flagd:v0.6.4 as flagd FROM busybox:1.36 COPY --from=flagd /flagd-build /flagd COPY flags/* . COPY scripts/* . -LABEL org.opencontainers.image.source = "https://github.com/open-feature/test-harness" +LABEL org.opencontainers.image.source = "https://github.com/open-feature/flagd-testbed" -ENTRYPOINT ["sh", "wrapper.sh", "./flagd", "start", "-f", "file:testing-flags.json", "-f", "file:changing-flag.json"] +ENTRYPOINT ["sh", "wrapper.sh", "./flagd", "start", \ + "-f", "file:testing-flags.json", \ + "-f", "file:changing-flag.json", \ + "-f", "file:custom-ops.json", \ + "-f", "file:evaluator-refs.json", \ + "-f", "file:zero-flags.json"] diff --git a/flags/custom-ops.json b/flags/custom-ops.json new file mode 100644 index 0000000..773da02 --- /dev/null +++ b/flags/custom-ops.json @@ -0,0 +1,104 @@ +{ + "flags": { + "fractional-flag": { + "state": "ENABLED", + "variants": { + "clubs": "clubs", + "diamonds": "diamonds", + "hearts": "hearts", + "spades": "spades", + "wild": "wild" + }, + "defaultVariant": "wild", + "targeting": { + "fractional": [ + { "var": "user.name" }, + [ "clubs", 25 ], + [ "diamonds", 25 ], + [ "hearts", 25 ], + [ "spades", 25 ] + ] + } + }, + "starts-ends-flag": { + "state": "ENABLED", + "variants": { + "prefix": "prefix", + "postfix": "postfix", + "none": "none" + }, + "defaultVariant": "none", + "targeting": { + "if": [ + { + "starts_with": [{"var": "id"}, "abc"] + }, + "prefix", { + "if": [ + { + "ends_with": [{"var": "id"}, "xyz"] + }, + "postfix", "none" + ] + } + ] + } + }, + "equal-greater-lesser-version-flag": { + "state": "ENABLED", + "variants": { + "equal": "equal", + "greater": "greater", + "lesser": "lesser", + "none": "none" + }, + "defaultVariant": "none", + "targeting": { + "if": [ + { + "sem_ver": [{"var": "version"}, "=", "2.0.0"] + }, + "equal", { + "if": [ + { + "sem_ver": [{"var": "version"}, ">", "2.0.0"] + }, + "greater", { + "if": [ + { + "sem_ver": [{"var": "version"}, "<", "2.0.0"] + }, + "lesser", "none" + ] + } + ] + } + ] + } + }, + "major-minor-version-flag": { + "state": "ENABLED", + "variants": { + "minor": "minor", + "major": "major", + "none": "none" + }, + "defaultVariant": "none", + "targeting": { + "if": [ + { + "sem_ver": [{"var": "version"}, "~", "3.0.0"] + }, + "minor", { + "if": [ + { + "sem_ver": [{"var": "version"}, "^", "3.0.0"] + }, + "major", "none" + ] + } + ] + } + } + } +} diff --git a/flags/evaluator-refs.json b/flags/evaluator-refs.json new file mode 100644 index 0000000..faad954 --- /dev/null +++ b/flags/evaluator-refs.json @@ -0,0 +1,52 @@ +{ + "flags": { + "some-email-targeted-flag": { + "state": "ENABLED", + "variants": { + "hi": "hi", + "bye": "bye", + "none": "none" + }, + "defaultVariant": "none", + "targeting": { + "if": [ + { + "$ref": "is_ballmer" + }, + "hi", + "bye" + ] + } + }, + "some-other-email-targeted-flag": { + "state": "ENABLED", + "variants": { + "yes": "yes", + "no": "no", + "none": "none" + }, + "defaultVariant": "none", + "targeting": { + "if": [ + { + "$ref": "is_ballmer" + }, + "yes", + "no" + ] + } + } + }, + "$evaluators": { + "is_ballmer": { + "==": [ + "ballmer@macrosoft.com", + { + "var": [ + "email" + ] + } + ] + } + } +} diff --git a/flags/testing-flags.json b/flags/testing-flags.json index bcf4517..200140d 100644 --- a/flags/testing-flags.json +++ b/flags/testing-flags.json @@ -8,14 +8,6 @@ }, "defaultVariant": "on" }, - "boolean-flag-copy": { - "state": "ENABLED", - "variants": { - "on": true, - "off": false - }, - "defaultVariant": "on" - }, "string-flag": { "state": "ENABLED", "variants": { @@ -24,14 +16,6 @@ }, "defaultVariant": "greeting" }, - "string-flag-copy": { - "state": "ENABLED", - "variants": { - "greeting": "hi", - "parting": "bye" - }, - "defaultVariant": "greeting" - }, "integer-flag": { "state": "ENABLED", "variants": { @@ -40,14 +24,6 @@ }, "defaultVariant": "ten" }, - "integer-flag-copy": { - "state": "ENABLED", - "variants": { - "one": 1, - "ten": 10 - }, - "defaultVariant": "ten" - }, "float-flag": { "state": "ENABLED", "variants": { @@ -56,14 +32,6 @@ }, "defaultVariant": "half" }, - "float-flag-copy": { - "state": "ENABLED", - "variants": { - "tenth": 0.1, - "half": 0.5 - }, - "defaultVariant": "half" - }, "object-flag": { "state": "ENABLED", "variants": { @@ -76,18 +44,6 @@ }, "defaultVariant": "template" }, - "object-flag-copy": { - "state": "ENABLED", - "variants": { - "empty": {}, - "template": { - "showImages": true, - "title": "Check out these pics!", - "imagesPerPage": 100 - } - }, - "defaultVariant": "template" - }, "context-aware": { "state": "ENABLED", "variants": { diff --git a/flags/zero-flags.json b/flags/zero-flags.json new file mode 100644 index 0000000..ed6a9ac --- /dev/null +++ b/flags/zero-flags.json @@ -0,0 +1,36 @@ +{ + "flags": { + "boolean-zero-flag": { + "state": "ENABLED", + "variants": { + "zero": false, + "non-zero": true + }, + "defaultVariant": "zero" + }, + "string-zero-flag": { + "state": "ENABLED", + "variants": { + "zero": "", + "non-zero": "str" + }, + "defaultVariant": "zero" + }, + "integer-zero-flag": { + "state": "ENABLED", + "variants": { + "zero": 0, + "non-zero": 1 + }, + "defaultVariant": "zero" + }, + "float-zero-flag": { + "state": "ENABLED", + "variants": { + "zero": 0.0, + "non-zero": 1.0 + }, + "defaultVariant": "zero" + } + } +} diff --git a/gherkin/flagd-json-evaluator.feature b/gherkin/flagd-json-evaluator.feature new file mode 100644 index 0000000..5631613 --- /dev/null +++ b/gherkin/flagd-json-evaluator.feature @@ -0,0 +1,63 @@ +Feature: flagd json evaluation + + # This test suite contains scenarios to test the json-evaluation of flagd and flag-in-process providers. + # It's associated with the flags configured in flags/changing-flag.json, flags/zero-flags.json, flags/custom-ops.json and evaluator-refs.json. + # It should be used in conjunection with the suites supplied by the OpenFeature specification. + + Background: + Given a flagd provider is set + + # evaluator refs + Scenario Outline: Evaluator reuse + When a string flag with key is evaluated with default value "fallback" + And a context containing a key "email", with value "ballmer@macrosoft.com" + Then the returned value should be + Examples: + | key | value | + | some-email-targeted-flag | hi | + | some-other-email-targeted-flag | yes | + + # custom operators + Scenario Outline: Fractional operator + When a string flag with key "fractional-flag" is evaluated with default value "fallback" + And a context containing a nested property with outer key "user" and inner key "name", with value + Then the returned value should be + Examples: + | name | value | + | jack | clubs | + | queen | diamonds | + | ace | hearts | + | joker | spades | + + Scenario Outline: Substring operators + When a string flag with key "starts-ends-flag" is evaluated with default value "fallback" + And a context containing a key "id", with value + Then the returned value should be + Examples: + | id | value | + | abcdef | prefix | + | uvwxyz | postfix | + | abcxyz | prefix | + | lmnopq | nomatch | + + Scenario Outline: Semantic version operator numeric comparision + When a string flag with key "equal-greater-lesser-version-flag" is evaluated with default value "fallback" + And a context containing a key "version", with value + Then the returned value should be + Examples: + | version | value | + | 2.0.0 | equal | + | 2.1.0 | greater | + | 1.9.0 | lesser | + | 2.0.0-alpha | lesser | + | 2.0.0.0 | invalid | + + Scenario Outline: Semantic version operator semantic comparision + When a string flag with key "major-minor-version-flag" is evaluated with default value "fallback" + And a context containing a key "version", with value + Then the returned value should be + Examples: + | version | value | + | 3.0.1 | minor | + | 3.1.0 | major | + | 4.0.0 | none | diff --git a/gherkin/flagd.feature b/gherkin/flagd.feature new file mode 100644 index 0000000..d2cecb8 --- /dev/null +++ b/gherkin/flagd.feature @@ -0,0 +1,36 @@ +Feature: flagd providers + + # This test suite contains scenarios to test flagd providers. + # It's associated with the flags configured in flags/changing-flag.json and flags/zero-flags.json. + # It should be used in conjunection with the suites supplied by the OpenFeature specification. + + Background: + Given a flagd provider is set + + # events + Scenario: Provider ready event + When a PROVIDER_READY handler is added + Then the PROVIDER_READY handler must run + + Scenario: Flag change event + When a PROVIDER_CONFIGURATION_CHANGED handler is added + And a flag with key "changing-flag" is modified + Then the PROVIDER_CONFIGURATION_CHANGED handler must run + And the event details must indicate "changing-flag" was altered + + # zero evaluation + Scenario: Resolves boolean zero value + When a boolean flag with key "boolean-zero-flag" is evaluated with default value "true" + Then the resolved boolean value should be "false" + + Scenario: Resolves string zero value + When a string flag with key "string-zero-flag" is evaluated with default value "hi" + Then the resolved string value should be "" + + Scenario: Resolves integer zero value + When an integer flag with key "integer-zero-flag" is evaluated with default value 1 + Then the resolved integer value should be 0 + + Scenario: Resolves float zero value + When a float flag with key "float-zero-flag" is evaluated with default value 0.1 + Then the resolved float value should be 0.0 diff --git a/sync/Dockerfile b/sync/Dockerfile index 7e0c423..6fa5d55 100644 --- a/sync/Dockerfile +++ b/sync/Dockerfile @@ -11,6 +11,11 @@ FROM busybox:1.36 COPY --from=builder /src/sync . COPY flags/* . COPY scripts/* . -LABEL org.opencontainers.image.source = "https://github.com/open-feature/test-harness" +LABEL org.opencontainers.image.source = "https://github.com/open-feature/flagd-testbed" -ENTRYPOINT ["sh", "wrapper.sh", "./sync", "start", "-f", "testing-flags.json", "-f", "changing-flag.json"] +ENTRYPOINT ["sh", "wrapper.sh", "./sync", "start", \ + "-f", "testing-flags.json", \ + "-f", "changing-flag.json", \ + "-f", "custom-ops.json", \ + "-f", "evaluator-refs.json", \ + "-f", "zero-flags.json"]