Skip to content

Commit

Permalink
Support initializable retriever (#1427)
Browse files Browse the repository at this point in the history
* Support initializable retriever

Signed-off-by: Thomas Poignant <[email protected]>

* Update feature_flag_test.go

Co-authored-by: Luiz Guilherme Ribeiro <[email protected]>

---------

Signed-off-by: Thomas Poignant <[email protected]>
Co-authored-by: Luiz Guilherme Ribeiro <[email protected]>
  • Loading branch information
thomaspoignant and luizgribeiro authored Jan 5, 2024
1 parent dc5f5d6 commit fd90988
Show file tree
Hide file tree
Showing 6 changed files with 234 additions and 14 deletions.
44 changes: 30 additions & 14 deletions feature_flag.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,11 @@ func Close() {
// GoFeatureFlag is the main object of the library
// it contains the cache, the config, the updater and the exporter.
type GoFeatureFlag struct {
cache cache.Manager
config Config
bgUpdater backgroundUpdater
dataExporter *exporter.Scheduler
cache cache.Manager
config Config
bgUpdater backgroundUpdater
dataExporter *exporter.Scheduler
retrieverManager *retriever.Manager
}

// ff is the default object for go-feature-flag
Expand Down Expand Up @@ -85,7 +86,17 @@ func New(config Config) (*GoFeatureFlag, error) {
goFF.bgUpdater = newBackgroundUpdater(config.PollingInterval, config.EnablePollingJitter)
goFF.cache = cache.New(notificationService, config.Logger)

err := retrieveFlagsAndUpdateCache(goFF.config, goFF.cache)
retrievers, err := config.GetRetrievers()
if err != nil {
return nil, err
}
goFF.retrieverManager = retriever.NewManager(config.Context, retrievers)
err = goFF.retrieverManager.Init(config.Context)
if err != nil && !config.StartWithRetrieverError {
return nil, fmt.Errorf("impossible to initialize the retrievers, please check your configuration: %v", err)
}

err = retrieveFlagsAndUpdateCache(goFF.config, goFF.cache, goFF.retrieverManager)
if err != nil && !config.StartWithRetrieverError {
return nil, fmt.Errorf("impossible to retrieve the flags, please check your configuration: %v", err)
}
Expand Down Expand Up @@ -119,6 +130,9 @@ func (g *GoFeatureFlag) Close() {
if g.dataExporter != nil {
g.dataExporter.Close()
}
if g.retrieverManager != nil {
_ = g.retrieverManager.Shutdown(g.config.Context)
}
}
}

Expand All @@ -127,7 +141,7 @@ func (g *GoFeatureFlag) startFlagUpdaterDaemon() {
for {
select {
case <-g.bgUpdater.ticker.C:
err := retrieveFlagsAndUpdateCache(g.config, g.cache)
err := retrieveFlagsAndUpdateCache(g.config, g.cache, g.retrieverManager)
if err != nil {
fflog.Printf(g.config.Logger, "error while updating the cache: %v\n", err)
}
Expand All @@ -138,13 +152,8 @@ func (g *GoFeatureFlag) startFlagUpdaterDaemon() {
}

// retrieveFlagsAndUpdateCache is called every X seconds to refresh the cache flag.
func retrieveFlagsAndUpdateCache(config Config, cache cache.Manager) error {
retrievers, err := config.GetRetrievers()
if err != nil {
fflog.Printf(config.Logger, "error while getting the file retriever: %v", err)
return err
}

func retrieveFlagsAndUpdateCache(config Config, cache cache.Manager, retrieverManager *retriever.Manager) error {
retrievers := retrieverManager.GetRetrievers()
// Results is the type that will receive the results when calling
// all the retrievers.
type Results struct {
Expand All @@ -169,6 +178,13 @@ func retrieveFlagsAndUpdateCache(config Config, cache cache.Manager) error {
// Launching GO routines to retrieve all files in parallel.
go func(r retriever.Retriever, format string, index int, ctx context.Context) {
defer wg.Done()

// If the retriever is not ready, we ignore it
if rr, ok := r.(retriever.InitializableRetriever); ok && rr.Status() != retriever.RetrieverReady {
resultsChan <- Results{Error: nil, Value: map[string]dto.DTO{}, Index: index}
return
}

rawValue, err := r.Retrieve(ctx)
if err != nil {
resultsChan <- Results{Error: err, Value: nil, Index: index}
Expand All @@ -195,7 +211,7 @@ func retrieveFlagsAndUpdateCache(config Config, cache cache.Manager) error {
}
}

err = cache.UpdateCache(newFlags, config.Logger)
err := cache.UpdateCache(newFlags, config.Logger)
if err != nil {
log.Printf("error: impossible to update the cache of the flags: %v", err)
return err
Expand Down
42 changes: 42 additions & 0 deletions feature_flag_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package ffclient_test

import (
"errors"
"github.com/thomaspoignant/go-feature-flag/ffcontext"
"github.com/thomaspoignant/go-feature-flag/testutils/initializableretriever"
"log"
"os"
"testing"
Expand Down Expand Up @@ -374,6 +376,46 @@ func TestValidUseCaseBigFlagFile(t *testing.T) {
assert.False(t, hasUnknownFlag, "User should use default value if flag does not exists")
}

func TestInitializableRetrieverWithRetrieverReady(t *testing.T) {
f, err := os.CreateTemp("", "")
assert.NoError(t, err)
// we delete the fileTemp to be sure that the retriever will have to create the file
err = os.Remove(f.Name())
assert.NoError(t, err)

r := initializableretriever.NewMockInitializableRetriever(f.Name(), retriever.RetrieverReady)
gff, err := ffclient.New(ffclient.Config{
PollingInterval: 5 * time.Second,
Retriever: &r,
})
assert.NoError(t, err)
user := ffcontext.NewEvaluationContext("random-key")
hasTestFlag, _ := gff.BoolVariation("flag-xxxx-123", user, false)
assert.True(t, hasTestFlag, "User should have test flag")

gff.Close()
_, err = os.Stat(f.Name())
assert.True(t, errors.Is(err, os.ErrNotExist))
}
func TestInitializableRetrieverWithRetrieverNotReady(t *testing.T) {
f, err := os.CreateTemp("", "")
assert.NoError(t, err)
// we delete the fileTemp to be sure that the retriever will have to create the file
err = os.Remove(f.Name())
assert.NoError(t, err)

r := initializableretriever.NewMockInitializableRetriever(f.Name(), retriever.RetrieverNotReady)
gff, err := ffclient.New(ffclient.Config{
PollingInterval: 5 * time.Second,
Retriever: &r,
})
defer gff.Close()
assert.NoError(t, err)
user := ffcontext.NewEvaluationContext("random-key")
hasTestFlag, _ := gff.BoolVariation("flag-xxxx-123", user, false)
assert.False(t, hasTestFlag, "Should resolve to default value if retriever is not ready")
}

func TestGoFeatureFlag_GetCacheRefreshDate(t *testing.T) {
type fields struct {
pollingInterval time.Duration
Expand Down
72 changes: 72 additions & 0 deletions retriever/manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package retriever

import (
"context"
"fmt"
)

// Manager is a struct that managed the retrievers.
type Manager struct {
ctx context.Context
retrievers []Retriever
onErrorRetriever []Retriever
}

// NewManager create a new Manager.
func NewManager(ctx context.Context, retrievers []Retriever) *Manager {
return &Manager{
ctx: ctx,
retrievers: retrievers,
onErrorRetriever: make([]Retriever, 0),
}
}

// Init the retrievers.
// This function will call the Init function of the retrievers that implements the InitializableRetriever interface.
func (m *Manager) Init(ctx context.Context) error {
return m.initRetrievers(ctx, m.retrievers)
}

// initRetrievers is a helper function to initialize the retrievers.
func (m *Manager) initRetrievers(ctx context.Context, retrieversToInit []Retriever) error {
m.onErrorRetriever = make([]Retriever, 0)
for _, retriever := range retrieversToInit {
if r, ok := retriever.(InitializableRetriever); ok {
err := r.Init(ctx)
if err != nil {
m.onErrorRetriever = append(m.onErrorRetriever, retriever)
}
}
}
if len(m.onErrorRetriever) > 0 {
return fmt.Errorf("error while initializing the retrievers: %v", m.onErrorRetriever)
}
return nil
}

// Shutdown the retrievers.
// This function will call the Shutdown function of the retrievers that implements the InitializableRetriever interface.
func (m *Manager) Shutdown(ctx context.Context) error {
onErrorRetriever := make([]Retriever, 0)
for _, retriever := range m.retrievers {
if r, ok := retriever.(InitializableRetriever); ok {
err := r.Shutdown(ctx)
if err != nil {
onErrorRetriever = append(onErrorRetriever, retriever)
}
}
}
if len(onErrorRetriever) > 0 {
return fmt.Errorf("error while shutting down the retrievers: %v", onErrorRetriever)
}
return nil
}

// GetRetrievers return the retrievers.
// If an error occurred during the initialization of the retrievers, we will return the retrievers that are ready.
func (m *Manager) GetRetrievers() []Retriever {
if len(m.onErrorRetriever) > 0 {
_ = m.initRetrievers(m.ctx, m.onErrorRetriever)
}
return m.retrievers
}
19 changes: 19 additions & 0 deletions retriever/retriever.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,22 @@ type Retriever interface {
// Retrieve function is supposed to load the file and to return a []byte of your flag configuration file.
Retrieve(ctx context.Context) ([]byte, error)
}

// InitializableRetriever is an extended version of the retriever that can be initialized and shutdown.
type InitializableRetriever interface {
Retrieve(ctx context.Context) ([]byte, error)
Init(ctx context.Context) error
Shutdown(ctx context.Context) error
Status() Status
}

// Status is the status of the retriever.
// It can be used to check if the retriever is ready to be used.
// If not ready, we wi will not use it.
type Status = string

const (
RetrieverReady Status = "READY"
RetrieverNotReady Status = "NOT_READY"
RetrieverError Status = "ERROR"
)
51 changes: 51 additions & 0 deletions testutils/initializableretriever/retriever.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package initializableretriever

import (
"context"
"github.com/thomaspoignant/go-feature-flag/retriever"
"os"
)

func NewMockInitializableRetriever(path string, status retriever.Status) Retriever {
return Retriever{
context: context.Background(),
Path: path,
status: status,
}
}

// Retriever is a mock provider, that create a file as init step and delete it at shutdown.
type Retriever struct {
context context.Context
Path string
status retriever.Status
}

// Retrieve is reading the file and return the content
func (r *Retriever) Retrieve(_ context.Context) ([]byte, error) {
content, err := os.ReadFile(r.Path)
if err != nil {
return nil, err
}
return content, nil
}

func (r *Retriever) Init(_ context.Context) error {
yamlString := `flag-xxxx-123:
variations:
A: true
B: false
defaultRule:
variation: A`

yamlBytes := []byte(yamlString)
return os.WriteFile(r.Path, yamlBytes, 0600)
}

func (r *Retriever) Shutdown(_ context.Context) error {
return os.Remove(r.Path)
}

func (r *Retriever) Status() retriever.Status {
return r.status
}
20 changes: 20 additions & 0 deletions website/docs/go_module/store_file/custom.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ sidebar_position: 30

# Custom Retriever

## Simple retriever
To create a custom retriever you must have a `struct` that implements the [`Retriever`](https://pkg.go.dev/github.com/thomaspoignant/go-feature-flag/retriever/#Retriever) interface.

```go linenums="1"
Expand All @@ -16,3 +17,22 @@ The `Retrieve` function is supposed to load the file and to return a `[]byte` o

You can check existing `Retriever` *([file](https://github.com/thomaspoignant/go-feature-flag/blob/main/retriever/fileretriever/retriever.go),
[s3](https://github.com/thomaspoignant/go-feature-flag/blob/main/retriever/s3retriever/retriever.go), ...)* to have an idea on how to do build your own.

## Initializable retriever
Sometimes you need to initialize your retriever before using it.
For example, if you want to connect to a database, you need to initialize the connection before using it.

To help you with that, you can use the [`InitializableRetriever`](https://pkg.go.dev/github.com/thomaspoignant/go-feature-flag/retriever/#InitializableRetriever) interface.

The only difference with the `Retriever` interface is that the `Init` func of your retriever will be called at the start of the application and the `Shutdown` func will be called when closing GO Feature Flag.

```go
type InitializableRetriever interface {
Retrieve(ctx context.Context) ([]byte, error)
Init(ctx context.Context) error
Shutdown(ctx context.Context) error
Status() retriever.Status
}
```
To avoid any issue to call the `Retrieve` function before the `Init` function, you have to manage the status of your retriever.
GO Feature Flag will try to call the `Retrieve` function only if the status is `RetrieverStatusReady`.

0 comments on commit fd90988

Please sign in to comment.