New exporter for s3 that supports go-aws-sdk-v2 SDK (#986)

* Bump dev.openfeature.contrib.providers:go-feature-flag from 0.2.10 to 0.2.11 in /openfeature/provider_tests/java-integration-tests (#979) * support v2 in relay-proxy Signed-off-by: Thomas Poignant <[email protected]> * Fix lint Signed-off-by: Thomas Poignant <[email protected]> * Fixing tests Signed-off-by: Thomas Poignant <[email protected]> --------- Signed-off-by: Thomas Poignant <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
thomaspoignant · Aug 9, 2023 · 44a2419 · 44a2419
1 parent 98c7b28
commit 44a2419
Show file tree

Hide file tree

Showing 14 changed files with 518 additions and 105 deletions.
diff --git a/cmd/relayproxy/service/gofeatureflag.go b/cmd/relayproxy/service/gofeatureflag.go
@@ -2,6 +2,7 @@ package service
 
 import (
 	"fmt"
+	"github.com/thomaspoignant/go-feature-flag/exporter/s3exporterv2"
 	"github.com/thomaspoignant/go-feature-flag/exporter/sqsexporter"
 	"time"
 
@@ -10,7 +11,6 @@ import (
 	"github.com/thomaspoignant/go-feature-flag/exporter/fileexporter"
 	"github.com/thomaspoignant/go-feature-flag/exporter/gcstorageexporter"
 	"github.com/thomaspoignant/go-feature-flag/exporter/logsexporter"
-	"github.com/thomaspoignant/go-feature-flag/exporter/s3exporter"
 	"github.com/thomaspoignant/go-feature-flag/exporter/webhookexporter"
 	"github.com/thomaspoignant/go-feature-flag/notifier"
 	"github.com/thomaspoignant/go-feature-flag/notifier/slacknotifier"
@@ -26,6 +26,8 @@ import (
 	"go.uber.org/zap"
 	"golang.org/x/net/context"
 	"k8s.io/client-go/rest"
+
+	awsConf "github.com/aws/aws-sdk-go-v2/config"
 )
 
 func NewGoFeatureFlagClient(
@@ -128,16 +130,9 @@ func initRetriever(c *config.RetrieverConf) (retriever.Retriever, error) {
 			Timeout:        retrieverTimeout,
 		}, nil
 	case config.FileRetriever:
-		return &fileretriever.Retriever{
-			Path: c.Path,
-		}, nil
-
+		return &fileretriever.Retriever{Path: c.Path}, nil
 	case config.S3Retriever:
-		return &s3retriever.Retriever{
-			Bucket: c.Bucket,
-			Item:   c.Item,
-		}, nil
-
+		return &s3retriever.Retriever{Bucket: c.Bucket, Item: c.Item}, nil
 	case config.HTTPRetriever:
 		return &httpretriever.Retriever{
 			URL: c.URL,
@@ -146,30 +141,16 @@ func initRetriever(c *config.RetrieverConf) (retriever.Retriever, error) {
 					return config.DefaultRetriever.HTTPMethod
 				}
 				return c.HTTPMethod
-			}(),
-			Body:    c.HTTPBody,
-			Header:  c.HTTPHeaders,
-			Timeout: retrieverTimeout,
-		}, nil
-
+			}(), Body: c.HTTPBody, Header: c.HTTPHeaders, Timeout: retrieverTimeout}, nil
 	case config.GoogleStorageRetriever:
-		return &gcstorageretriever.Retriever{
-			Bucket: c.Bucket,
-			Object: c.Object,
-		}, nil
-
+		return &gcstorageretriever.Retriever{Bucket: c.Bucket, Object: c.Object}, nil
 	case config.KubernetesRetriever:
 		client, err := rest.InClusterConfig()
 		if err != nil {
 			return nil, err
 		}
-		return &k8sretriever.Retriever{
-			Namespace:     c.Namespace,
-			ConfigMapName: c.ConfigMap,
-			Key:           c.Key,
-			ClientConfig:  *client,
-		}, nil
-
+		return &k8sretriever.Retriever{Namespace: c.Namespace, ConfigMapName: c.ConfigMap, Key: c.Key,
+			ClientConfig: *client}, nil
 	default:
 		return nil, fmt.Errorf("invalid retriever: kind \"%s\" "+
 			"is not supported, accepted kind: [googleStorage, http, s3, file, github]", c.Kind)
@@ -181,22 +162,18 @@ func initExporter(c *config.ExporterConf) (ffclient.DataExporter, error) {
 	if c.Format != "" {
 		format = c.Format
 	}
-
 	filename := config.DefaultExporter.FileName
 	if c.Filename != "" {
 		filename = c.Filename
 	}
-
 	csvTemplate := config.DefaultExporter.CsvFormat
 	if c.CsvTemplate != "" {
 		csvTemplate = c.CsvTemplate
 	}
-
 	parquetCompressionCodec := config.DefaultExporter.ParquetCompressionCodec
 	if c.ParquetCompressionCodec != "" {
 		parquetCompressionCodec = c.ParquetCompressionCodec
 	}
-
 	dataExp := ffclient.DataExporter{
 		FlushInterval: func() time.Duration {
 			if c.FlushInterval != 0 {
@@ -217,12 +194,10 @@ func initExporter(c *config.ExporterConf) (ffclient.DataExporter, error) {
 		dataExp.Exporter = &webhookexporter.Exporter{
 			EndpointURL: c.EndpointURL, Secret: c.Secret, Meta: c.Meta, Headers: c.Headers}
 		return dataExp, nil
-
 	case config.FileExporter:
 		dataExp.Exporter = &fileexporter.Exporter{Format: format, OutputDir: c.OutputDir, Filename: filename,
 			CsvTemplate: csvTemplate, ParquetCompressionCodec: parquetCompressionCodec}
 		return dataExp, nil
-
 	case config.LogExporter:
 		dataExp.Exporter = &logsexporter.Exporter{
 			LogFormat: func() string {
@@ -233,18 +208,21 @@ func initExporter(c *config.ExporterConf) (ffclient.DataExporter, error) {
 			}(),
 		}
 		return dataExp, nil
-
 	case config.S3Exporter:
-		dataExp.Exporter = &s3exporter.Exporter{
+		awsConfig, err := awsConf.LoadDefaultConfig(context.Background())
+		if err != nil {
+			return dataExp, err
+		}
+		dataExp.Exporter = &s3exporterv2.Exporter{
 			Bucket:                  c.Bucket,
 			Format:                  format,
 			S3Path:                  c.Path,
 			Filename:                filename,
 			CsvTemplate:             csvTemplate,
 			ParquetCompressionCodec: parquetCompressionCodec,
+			AwsConfig:               &awsConfig,
 		}
 		return dataExp, nil
-
 	case config.GoogleStorageExporter:
 		dataExp.Exporter = &gcstorageexporter.Exporter{
 			Bucket:                  c.Bucket,
@@ -255,9 +233,12 @@ func initExporter(c *config.ExporterConf) (ffclient.DataExporter, error) {
 			ParquetCompressionCodec: parquetCompressionCodec,
 		}
 		return dataExp, nil
-
 	case config.SQSExporter:
-		dataExp.Exporter = &sqsexporter.Exporter{QueueURL: c.QueueURL}
+		awsConfig, err := awsConf.LoadDefaultConfig(context.Background())
+		if err != nil {
+			return dataExp, err
+		}
+		dataExp.Exporter = &sqsexporter.Exporter{QueueURL: c.QueueURL, AwsConfig: &awsConfig}
 		return dataExp, nil
 
 	default:

diff --git a/cmd/relayproxy/service/gofeatureflag_test.go b/cmd/relayproxy/service/gofeatureflag_test.go
@@ -2,6 +2,8 @@ package service
 
 import (
 	"fmt"
+	"github.com/thomaspoignant/go-feature-flag/exporter"
+	"github.com/thomaspoignant/go-feature-flag/exporter/s3exporterv2"
 	"github.com/thomaspoignant/go-feature-flag/exporter/sqsexporter"
 	"github.com/thomaspoignant/go-feature-flag/notifier"
 	"github.com/thomaspoignant/go-feature-flag/notifier/slacknotifier"
@@ -16,7 +18,6 @@ import (
 	"github.com/thomaspoignant/go-feature-flag/exporter/fileexporter"
 	"github.com/thomaspoignant/go-feature-flag/exporter/gcstorageexporter"
 	"github.com/thomaspoignant/go-feature-flag/exporter/logsexporter"
-	"github.com/thomaspoignant/go-feature-flag/exporter/s3exporter"
 	"github.com/thomaspoignant/go-feature-flag/exporter/webhookexporter"
 	"github.com/thomaspoignant/go-feature-flag/retriever"
 	"github.com/thomaspoignant/go-feature-flag/retriever/fileretriever"
@@ -178,10 +179,12 @@ func Test_initRetriever(t *testing.T) {
 
 func Test_initExporter(t *testing.T) {
 	tests := []struct {
-		name    string
-		conf    *config.ExporterConf
-		want    ffclient.DataExporter
-		wantErr assert.ErrorAssertionFunc
+		name                   string
+		conf                   *config.ExporterConf
+		want                   ffclient.DataExporter
+		wantErr                assert.ErrorAssertionFunc
+		wantType               exporter.Exporter
+		skipCompleteValidation bool
 	}{
 		{
 			name:    "Convert unknown Exporter",
@@ -207,6 +210,7 @@ func Test_initExporter(t *testing.T) {
 					Meta:        nil,
 				},
 			},
+			wantType: &webhookexporter.Exporter{},
 		},
 		{
 			name:    "Convert FileExporter",
@@ -227,6 +231,7 @@ func Test_initExporter(t *testing.T) {
 					ParquetCompressionCodec: parquet.CompressionCodec_UNCOMPRESSED.String(),
 				},
 			},
+			wantType: &fileexporter.Exporter{},
 		},
 		{
 			name:    "Convert LogExporter",
@@ -241,6 +246,7 @@ func Test_initExporter(t *testing.T) {
 					LogFormat: config.DefaultExporter.LogFormat,
 				},
 			},
+			wantType: &logsexporter.Exporter{},
 		},
 		{
 			name:    "Convert S3Exporter",
@@ -254,7 +260,7 @@ func Test_initExporter(t *testing.T) {
 			want: ffclient.DataExporter{
 				FlushInterval:    10 * time.Millisecond,
 				MaxEventInMemory: config.DefaultExporter.MaxEventInMemory,
-				Exporter: &s3exporter.Exporter{
+				Exporter: &s3exporterv2.Exporter{
 					Bucket:                  "my-bucket",
 					Format:                  config.DefaultExporter.Format,
 					S3Path:                  "/my-path/",
@@ -263,6 +269,8 @@ func Test_initExporter(t *testing.T) {
 					ParquetCompressionCodec: config.DefaultExporter.ParquetCompressionCodec,
 				},
 			},
+			wantType:               &s3exporterv2.Exporter{},
+			skipCompleteValidation: true,
 		},
 		{
 			name:    "Convert SQSExporter",
@@ -279,6 +287,8 @@ func Test_initExporter(t *testing.T) {
 					QueueURL: "https://sqs.eu-west-1.amazonaws.com/XXX/test-queue",
 				},
 			},
+			wantType:               &sqsexporter.Exporter{},
+			skipCompleteValidation: true,
 		},
 		{
 			name:    "Convert GoogleStorageExporter",
@@ -301,13 +311,15 @@ func Test_initExporter(t *testing.T) {
 					ParquetCompressionCodec: config.DefaultExporter.ParquetCompressionCodec,
 				},
 			},
+			wantType: &gcstorageexporter.Exporter{},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := initExporter(tt.conf)
 			tt.wantErr(t, err)
-			if err == nil {
+			assert.IsType(t, tt.wantType, got.Exporter)
+			if err == nil && !tt.skipCompleteValidation {
 				assert.Equal(t, tt.want, got)
 			}
 		})

diff --git a/examples/data_export_s3/main.go b/examples/data_export_s3/main.go
@@ -2,16 +2,14 @@ package main
 
 import (
 	"context"
+	"github.com/aws/aws-sdk-go-v2/config"
+	ffclient "github.com/thomaspoignant/go-feature-flag"
+	"github.com/thomaspoignant/go-feature-flag/exporter/s3exporterv2"
 	"github.com/thomaspoignant/go-feature-flag/ffcontext"
+	"github.com/thomaspoignant/go-feature-flag/retriever/fileretriever"
 	"log"
 	"os"
 	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/thomaspoignant/go-feature-flag/exporter/s3exporter"
-	"github.com/thomaspoignant/go-feature-flag/retriever/fileretriever"
-
-	ffclient "github.com/thomaspoignant/go-feature-flag"
 )
 
 func main() {
@@ -23,6 +21,7 @@ func main() {
 			   - /go-feature-flag/variations/flag-variation-EXAMPLE-<timestamp>.json
 			   - /go-feature-flag/variations/flag-variation-EXAMPLE-<timestamp>.json
 	*/
+	awsConfig, _ := config.LoadDefaultConfig(context.Background())
 	err := ffclient.Init(ffclient.Config{
 		PollingInterval: 10 * time.Second,
 		Logger:          log.New(os.Stdout, "", 0),
@@ -33,14 +32,12 @@ func main() {
 		DataExporter: ffclient.DataExporter{
 			FlushInterval:    1 * time.Second,
 			MaxEventInMemory: 100,
-			Exporter: &s3exporter.Exporter{
-				Format:   "json",
-				Bucket:   "my-test-bucket",
-				S3Path:   "/go-feature-flag/variations/",
-				Filename: "flag-variation-{{ .Timestamp}}.{{ .Format}}",
-				AwsConfig: &aws.Config{
-					Region: aws.String("eu-west-1"),
-				},
+			Exporter: &s3exporterv2.Exporter{
+				Format:    "json",
+				Bucket:    "my-test-bucket",
+				S3Path:    "/go-feature-flag/variations/",
+				Filename:  "flag-variation-{{ .Timestamp}}.{{ .Format}}",
+				AwsConfig: &awsConfig,
 			},
 		},
 	})

diff --git a/exporter/s3exporter/exporter.go b/exporter/s3exporter/exporter.go
@@ -16,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
 )
 
+// Deprecated: Please use s3exporterv2.Exporter instead, it will use the go-aws-sdk-v2.
 type Exporter struct {
 	// Bucket is the name of your Exporter Bucket.
 	Bucket string