diff --git a/.gitignore b/.gitignore index 2d2b47d..022119e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea -node_modules \ No newline at end of file +node_modules +flags/changing-flag.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4e3a986..aa40ab6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ -FROM ghcr.io/open-feature/flagd:v0.5.2 +FROM ghcr.io/open-feature/flagd:v0.6.3 as flagd -COPY testing-flags.json testing-flags.json +FROM busybox:1.36 + +COPY --from=flagd /flagd-build /flagd +COPY flags/* . +COPY wrapper.sh . LABEL org.opencontainers.image.source = "https://github.com/open-feature/test-harness" -ENTRYPOINT ["/flagd", "start", "-f", "file:testing-flags.json"] +ENTRYPOINT ["sh", "wrapper.sh"] diff --git a/README.md b/README.md index 4ca0a1a..bf2c6f7 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,9 @@ # OpenFeature Test Harness -This repository contains the makings of an end-to-end test suite for OpenFeature SDKs. +This repository contains a docker image to support the [gherkin suites](https://github.com/open-feature/spec/blob/main/specification/appendix-b-gherkin-suites.md) in the OpenFeature specification. ## flagd-testbed container The _flagd-testbed_ container is a docker image built on flagd, which essentially just adds a simple set of flags for the purposes of testing OpenFeature SDKs. - -## Gherkin test suite - -The test suite is a set of [_gherkin_](https://cucumber.io/docs/gherkin/) tests that define expected behavior associated with the flags defined in the flagd-testbed. Combined with the _flagd-provider_ for that SDK and the flagd-testbed, these comprise a complete end-to-end tests suite. - -### Lint Gherkin files - -The Gherkin files structure can be linted using [gherkin-lint](https://github.com/vsiakka/gherkin-lint). The following commands require Node.js 10 or later. - -1. npm install -1. npm run gherkin-lint +`testing-flags.json` contains a set of flags consistent with the [evaluation feature](https://github.com/open-feature/spec/blob/main/specification/assets/gherkin/evaluation.feature). +`change-flag.sh` runs in the test container generates a file `changing-flag.json`, which contains a flag that changes once every seconds, allowing to easily test change events. \ No newline at end of file diff --git a/features/caching.feature b/features/caching.feature deleted file mode 100644 index 6be9205..0000000 --- a/features/caching.feature +++ /dev/null @@ -1,88 +0,0 @@ -Feature: Flag caching - -# This test suite contains scenarios relating to caching, particularly when an SDK is configured to use a flagd provider. Not all providers may observe these caching semantics. - - Background: - Given a provider is registered with cache enabled - - # caches value - Scenario: Caches boolean value - Given a boolean flag with key "boolean-flag" is evaluated with details and default value "false" - When a boolean flag with key "boolean-flag" is evaluated with details and default value "false" - Then the resolved boolean details reason should be "CACHED" - - Scenario: Caches string value - Given a string flag with key "string-flag" is evaluated with details and default value "bye" - When a string flag with key "string-flag" is evaluated with details and default value "bye" - Then the resolved string details reason should be "CACHED" - - Scenario: Caches integer value - Given an integer flag with key "integer-flag" is evaluated with details and default value 1 - When an integer flag with key "integer-flag" is evaluated with details and default value 1 - Then the resolved integer details reason should be "CACHED" - - Scenario: Caches float value - Given a float flag with key "float-flag" is evaluated with details and default value 0.1 - When a float flag with key "float-flag" is evaluated with details and default value 0.1 - Then the resolved float details reason should be "CACHED" - - Scenario: Caches object value - Given an object flag with key "object-flag" is evaluated with details and a null default value - When an object flag with key "object-flag" is evaluated with details and a null default value - Then the resolved object details reason should be "CACHED" - - # invalidates cached value - Scenario: Invalidates cache on boolean flag configuration update - Given a boolean flag with key "boolean-flag" is evaluated with details and default value "false" - And a boolean flag with key "boolean-flag-copy" is evaluated with details and default value "false" - When the flag's configuration with key "boolean-flag" is updated to defaultVariant "off" - And sleep for 1000 milliseconds - And a boolean flag with key "boolean-flag" is evaluated with details and default value "false" - And a boolean flag with key "boolean-flag-copy" is evaluated with details and default value "false" - Then the resolved boolean details reason of flag with key "boolean-flag" should be "STATIC" - And the resolved boolean details reason of flag with key "boolean-flag-copy" should be "CACHED" - - Scenario: Invalidates cache on string flag configuration update - Given a string flag with key "string-flag" is evaluated with details and default value "bye" - And a string flag with key "string-flag-copy" is evaluated with details and default value "bye" - When the flag's configuration with key "string-flag" is updated to defaultVariant "parting" - And sleep for 1000 milliseconds - And a string flag with key "string-flag" is evaluated with details and default value "bye" - And a string flag with key "string-flag-copy" is evaluated with details and default value "bye" - Then the resolved string details reason of flag with key "string-flag" should be "STATIC" - And the resolved string details reason of flag with key "string-flag-copy" should be "CACHED" - - Scenario: Invalidates cache on integer flag configuration update - Given an integer flag with key "integer-flag" is evaluated with details and default value 1 - And an integer flag with key "integer-flag-copy" is evaluated with details and default value 1 - When the flag's configuration with key "integer-flag" is updated to defaultVariant "one" - And sleep for 1000 milliseconds - And an integer flag with key "integer-flag" is evaluated with details and default value 1 - And an integer flag with key "integer-flag-copy" is evaluated with details and default value 1 - Then the resolved integer details reason of flag with key "integer-flag" should be "STATIC" - And the resolved integer details reason of flag with key "integer-flag-copy" should be "CACHED" - - Scenario: Invalidates cache on float flag configuration update - Given a float flag with key "float-flag" is evaluated with details and default value 0.1 - And a float flag with key "float-flag-copy" is evaluated with details and default value 0.1 - When the flag's configuration with key "float-flag" is updated to defaultVariant "tenth" - And sleep for 1000 milliseconds - And a float flag with key "float-flag" is evaluated with details and default value 0.1 - And a float flag with key "float-flag-copy" is evaluated with details and default value 0.1 - Then the resolved float details reason of flag with key "float-flag" should be "STATIC" - And the resolved float details reason of flag with key "float-flag-copy" should be "CACHED" - - Scenario: Invalidates cache on object flag configuration update - Given an object flag with key "object-flag" is evaluated with details and a null default value - And an object flag with key "object-flag-copy" is evaluated with details and a null default value - When the flag's configuration with key "object-flag" is updated to defaultVariant "empty" - And sleep for 1000 milliseconds - And an object flag with key "object-flag" is evaluated with details and a null default value - And an object flag with key "object-flag-copy" is evaluated with details and a null default value - Then the resolved object details reason of flag with key "object-flag" should be "STATIC" - And the resolved object details reason of flag with key "object-flag-copy" should be "CACHED" - - Scenario: Non-static flag not cached - Given a string flag with key "context-aware" is evaluated with details and default value "EXTERNAL" - When a string flag with key "context-aware" is evaluated with details and default value "EXTERNAL" - Then the resolved string details reason should be "TARGETING_MATCH" diff --git a/features/evaluation.feature b/features/evaluation.feature deleted file mode 100644 index e12ef54..0000000 --- a/features/evaluation.feature +++ /dev/null @@ -1,67 +0,0 @@ -Feature: Flag evaluation - -# This test suite contains scenarios to test the flag evaluation API. - - Background: - Given a provider is registered with cache disabled - - # basic evaluation - Scenario: Resolves boolean value - When a boolean flag with key "boolean-flag" is evaluated with default value "false" - Then the resolved boolean value should be "true" - - Scenario: Resolves string value - When a string flag with key "string-flag" is evaluated with default value "bye" - Then the resolved string value should be "hi" - - Scenario: Resolves integer value - When an integer flag with key "integer-flag" is evaluated with default value 1 - Then the resolved integer value should be 10 - - Scenario: Resolves float value - When a float flag with key "float-flag" is evaluated with default value 0.1 - Then the resolved float value should be 0.5 - - Scenario: Resolves object value - When an object flag with key "object-flag" is evaluated with a null default value - Then the resolved object value should be contain fields "showImages", "title", and "imagesPerPage", with values "true", "Check out these pics!" and 100, respectively - - # detailed evaluation - Scenario: Resolves boolean details - When a boolean flag with key "boolean-flag" is evaluated with details and default value "false" - Then the resolved boolean details value should be "true", the variant should be "on", and the reason should be "STATIC" - - Scenario: Resolves string details - When a string flag with key "string-flag" is evaluated with details and default value "bye" - Then the resolved string details value should be "hi", the variant should be "greeting", and the reason should be "STATIC" - - Scenario: Resolves integer details - When an integer flag with key "integer-flag" is evaluated with details and default value 1 - Then the resolved integer details value should be 10, the variant should be "ten", and the reason should be "STATIC" - - Scenario: Resolves float details - When a float flag with key "float-flag" is evaluated with details and default value 0.1 - Then the resolved float details value should be 0.5, the variant should be "half", and the reason should be "STATIC" - - Scenario: Resolves object details - When an object flag with key "object-flag" is evaluated with details and a null default value - Then the resolved object details value should be contain fields "showImages", "title", and "imagesPerPage", with values "true", "Check out these pics!" and 100, respectively - And the variant should be "template", and the reason should be "STATIC" - - # context-aware evaluation - Scenario: Resolves based on context - When context contains keys "fn", "ln", "age", "customer" with values "Sulisław", "Świętopełk", 29, "false" - And a flag with key "context-aware" is evaluated with default value "EXTERNAL" - Then the resolved string response should be "INTERNAL" - And the resolved flag value is "EXTERNAL" when the context is empty - - # errors - Scenario: Flag not found - When a non-existent string flag with key "missing-flag" is evaluated with details and a default value "uh-oh" - Then the default string value should be returned - And the reason should indicate an error and the error code should indicate a missing flag with "FLAG_NOT_FOUND" - - Scenario: Type error - When a string flag with key "wrong-flag" is evaluated as an integer, with details and a default value 13 - Then the default integer value should be returned - And the reason should indicate an error and the error code should indicate a type mismatch with "TYPE_MISMATCH" diff --git a/flags/change-flag.sh b/flags/change-flag.sh new file mode 100644 index 0000000..31802a9 --- /dev/null +++ b/flags/change-flag.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# this simple script toggles a flag value for a easy way of testing events +cat ./changing-flag-foo.json > ./changing-flag.json + +while true +do + sleep 1 + cat ./changing-flag-foo.json > ./changing-flag.json + echo 'updated flag changing-flag value to "foo"' + + sleep 1 + cat ./changing-flag-bar.json > ./changing-flag.json + echo 'updated flag changing-flag value to "bar"' +done \ No newline at end of file diff --git a/flags/changing-flag-bar.json b/flags/changing-flag-bar.json new file mode 100644 index 0000000..411b7e9 --- /dev/null +++ b/flags/changing-flag-bar.json @@ -0,0 +1,12 @@ +{ + "flags": { + "changing-flag": { + "state": "ENABLED", + "variants": { + "foo": "foo", + "bar": "bar" + }, + "defaultVariant": "bar" + } + } +} diff --git a/flags/changing-flag-foo.json b/flags/changing-flag-foo.json new file mode 100644 index 0000000..c82a63e --- /dev/null +++ b/flags/changing-flag-foo.json @@ -0,0 +1,12 @@ +{ + "flags": { + "changing-flag": { + "state": "ENABLED", + "variants": { + "foo": "foo", + "bar": "bar" + }, + "defaultVariant": "foo" + } + } +} diff --git a/testing-flags.json b/flags/testing-flags.json similarity index 100% rename from testing-flags.json rename to flags/testing-flags.json diff --git a/wrapper.sh b/wrapper.sh new file mode 100644 index 0000000..46bfb2c --- /dev/null +++ b/wrapper.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# wrapper script to start change-flag.sh and flagd, and forward signals to the child process + +# handle SIGINTs and SIGTERMs so we can kill the container +handle_term() { + kill -TERM "$child" 2>/dev/null +} + +handle_int() { + kill -INT "$child" 2>/dev/null +} + +trap handle_term SIGTERM +trap handle_int SIGINT + +# start change scriptm and flagd +sh ./change-flag.sh & +./flagd start -f 'file:testing-flags.json' -f 'file:changing-flag.json' & + +# wait on flagd +child=$! +wait "$child" \ No newline at end of file