Update to latest dskit (new flagext.Secret and move of pkg/tenant) (g…

…rafana#1356) * Update dskit * Refactor to make use of new secret flag type * Use tenant package from dskit * Add CHANGELOG entry
anovateam · Apr 5, 2022 · 464dcbf · 464dcbf
1 parent b79e432
commit 464dcbf
Show file tree

Hide file tree

Showing 45 changed files with 609 additions and 435 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
   **BREAKING CHANGE** Dropped `tempo-search-retention-duration` parameter.  [#1297](https://github.com/grafana/tempo/pull/1297) (@joe-elliott)
 * [CHANGE] Updated storage.trace.pool.queue_depth default from 200->10000. [#1345](https://github.com/grafana/tempo/pull/1345) (@joe-elliott)
 * [CHANGE] Update alpine images to 3.15 [#1330](https://github.com/grafana/tempo/pull/1330) (@zalegrala)
+* [CHANGE] Updated flags `-storage.trace.azure.storage-account-name` and `-storage.trace.s3.access_key` to no longer to be considered as secrets [#1356](https://github.com/grafana/tempo/pull/1356) (@simonswine)
 * [FEATURE]: v2 object encoding added. This encoding adds a start/end timestamp to every record to reduce proto marshalling and increase search speed.  
   **BREAKING CHANGE** After this rollout the distributors will use a new API on the ingesters. As such you must rollout all ingesters before rolling the 
   distributors. Also, during this period, the ingesters will use considerably more resources and as such should be scaled up (or incoming traffic should be

diff --git a/cmd/tempo-serverless/cloud-functions/go.sum b/cmd/tempo-serverless/cloud-functions/go.sum
@@ -1092,8 +1092,9 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/dskit v0.0.0-20211021180445-3bd016e9d7f1/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM=
-github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d h1:YwUtZIQFjlH6e2b5dFLfW1h/vTkTXNkZqv9qeU8b5h0=
 github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d/go.mod h1:M0/dlftwBvH7+hdNNpjMa/CUXD7gsew67mbkCuDlFXE=
+github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca h1:0qHzm6VS0bCsSWKHuyfpt+pdpyScdZbzY/IFIyKSYOk=
+github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca/go.mod h1:q51XdMLLHNZJSG6KOGujC20ed2OoLFdx0hBmOEVfRs0=
 github.com/grafana/e2e v0.1.0/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo=
 github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280 h1:MOND6wXrwVXEzmL2bZ+Jcbgycwt1LD5q6NQbqz/Nlic=
 github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=

diff --git a/cmd/tempo-serverless/handler.go b/cmd/tempo-serverless/handler.go
@@ -212,9 +212,7 @@ func stringToFlagExt() mapstructure.DecodeHookFunc {
 		if t != reflect.TypeOf(flagext.Secret{}) {
 			return data, nil
 		}
-		return flagext.Secret{
-			Value: data.(string),
-		}, nil
+		return flagext.SecretWithValue(data.(string)), nil
 	}
 }
 

diff --git a/cmd/tempo-serverless/handler_test.go b/cmd/tempo-serverless/handler_test.go
@@ -53,7 +53,7 @@ func TestLoadConfigS3(t *testing.T) {
 	if cfg.S3.Endpoint != "glerg" {
 		t.Error("s3 endpoint should be glerg", cfg.S3.Endpoint)
 	}
-	if cfg.S3.AccessKey.String() != "access" {
+	if cfg.S3.AccessKey != "access" {
 		t.Error("s3 access key should be access", cfg.S3.AccessKey)
 	}
 	if cfg.S3.SecretKey.String() != "secret" {

diff --git a/cmd/tempo-serverless/lambda/go.sum b/cmd/tempo-serverless/lambda/go.sum
@@ -1084,8 +1084,9 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/dskit v0.0.0-20211021180445-3bd016e9d7f1/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM=
-github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d h1:YwUtZIQFjlH6e2b5dFLfW1h/vTkTXNkZqv9qeU8b5h0=
 github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d/go.mod h1:M0/dlftwBvH7+hdNNpjMa/CUXD7gsew67mbkCuDlFXE=
+github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca h1:0qHzm6VS0bCsSWKHuyfpt+pdpyScdZbzY/IFIyKSYOk=
+github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca/go.mod h1:q51XdMLLHNZJSG6KOGujC20ed2OoLFdx0hBmOEVfRs0=
 github.com/grafana/e2e v0.1.0/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo=
 github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280 h1:MOND6wXrwVXEzmL2bZ+Jcbgycwt1LD5q6NQbqz/Nlic=
 github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=

diff --git a/go.mod b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/aws/aws-sdk-go v1.43.4
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
 	github.com/cespare/xxhash v1.1.0
+	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cristalhq/hedgedhttp v0.7.0
 	github.com/drone/envsubst v1.0.3
 	github.com/dustin/go-humanize v1.0.0
@@ -28,7 +29,7 @@ require (
 	github.com/google/go-cmp v0.5.7
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
-	github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d
+	github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca
 	github.com/grafana/e2e v0.1.0
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/hashicorp/go-hclog v1.0.0
@@ -102,7 +103,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coreos/etcd v3.3.25+incompatible // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect

diff --git a/go.sum b/go.sum
@@ -1128,8 +1128,9 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/dskit v0.0.0-20211021180445-3bd016e9d7f1/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM=
-github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d h1:YwUtZIQFjlH6e2b5dFLfW1h/vTkTXNkZqv9qeU8b5h0=
 github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d/go.mod h1:M0/dlftwBvH7+hdNNpjMa/CUXD7gsew67mbkCuDlFXE=
+github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca h1:0qHzm6VS0bCsSWKHuyfpt+pdpyScdZbzY/IFIyKSYOk=
+github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca/go.mod h1:q51XdMLLHNZJSG6KOGujC20ed2OoLFdx0hBmOEVfRs0=
 github.com/grafana/e2e v0.1.0 h1:nThd0U0TjUqyOOupSb+qDd4BOdhqwhR/oYbjoqiMlZk=
 github.com/grafana/e2e v0.1.0/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo=
 github.com/grafana/opentelemetry-collector v0.4.1-0.20211201123047-86905635e8f8 h1:jZAKyumYgG79/hCc+PKux00HpbzXcbjN1wc5fO4p4bU=

diff --git a/modules/frontend/transport/handler.go b/modules/frontend/transport/handler.go
@@ -15,13 +15,13 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/grafana/dskit/tenant"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/httpgrpc/server"
 
 	querier_stats "github.com/grafana/tempo/modules/querier/stats"
-	"github.com/grafana/tempo/pkg/tenant"
 	"github.com/grafana/tempo/pkg/util"
 	util_log "github.com/grafana/tempo/pkg/util/log"
 )

diff --git a/modules/frontend/v1/frontend.go b/modules/frontend/v1/frontend.go
@@ -10,6 +10,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/services"
+	"github.com/grafana/dskit/tenant"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -19,7 +20,6 @@ import (
 	"github.com/grafana/tempo/modules/frontend/v1/frontendv1pb"
 	"github.com/grafana/tempo/modules/querier/stats"
 	"github.com/grafana/tempo/pkg/scheduler/queue"
-	"github.com/grafana/tempo/pkg/tenant"
 	"github.com/grafana/tempo/pkg/util"
 	"github.com/grafana/tempo/pkg/util/httpgrpcutil"
 	"github.com/grafana/tempo/pkg/validation"

diff --git a/modules/frontend/v2/frontend.go b/modules/frontend/v2/frontend.go
@@ -14,6 +14,7 @@ import (
 	"github.com/grafana/dskit/flagext"
 	"github.com/grafana/dskit/grpcclient"
 	"github.com/grafana/dskit/services"
+	"github.com/grafana/dskit/tenant"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -23,7 +24,6 @@ import (
 
 	"github.com/grafana/tempo/modules/frontend/v2/frontendv2pb"
 	"github.com/grafana/tempo/modules/querier/stats"
-	"github.com/grafana/tempo/pkg/tenant"
 	"github.com/grafana/tempo/pkg/util/httpgrpcutil"
 )
 

diff --git a/modules/storage/config.go b/modules/storage/config.go
@@ -53,8 +53,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.Trace.Block.SearchPageSizeBytes = 1024 * 1024 // 1 MB
 
 	cfg.Trace.Azure = &azure.Config{}
-	f.StringVar(&cfg.Trace.Azure.StorageAccountName.Value, util.PrefixConfig(prefix, "trace.azure.storage-account-name"), "", "Azure storage account name.")
-	f.StringVar(&cfg.Trace.Azure.StorageAccountKey.Value, util.PrefixConfig(prefix, "trace.azure.storage-account-key"), "", "Azure storage access key.")
+	f.StringVar(&cfg.Trace.Azure.StorageAccountName, util.PrefixConfig(prefix, "trace.azure.storage-account-name"), "", "Azure storage account name.")
+	f.Var(&cfg.Trace.Azure.StorageAccountKey, util.PrefixConfig(prefix, "trace.azure.storage-account-key"), "Azure storage access key.")
 	f.StringVar(&cfg.Trace.Azure.ContainerName, util.PrefixConfig(prefix, "trace.azure.container-name"), "", "Azure container name to store blocks in.")
 	f.StringVar(&cfg.Trace.Azure.Endpoint, util.PrefixConfig(prefix, "trace.azure.endpoint"), "blob.core.windows.net", "Azure endpoint to push blocks to.")
 	f.IntVar(&cfg.Trace.Azure.MaxBuffers, util.PrefixConfig(prefix, "trace.azure.max-buffers"), 4, "Number of simultaneous uploads.")
@@ -64,8 +64,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.Trace.S3 = &s3.Config{}
 	f.StringVar(&cfg.Trace.S3.Bucket, util.PrefixConfig(prefix, "trace.s3.bucket"), "", "s3 bucket to store blocks in.")
 	f.StringVar(&cfg.Trace.S3.Endpoint, util.PrefixConfig(prefix, "trace.s3.endpoint"), "", "s3 endpoint to push blocks to.")
-	f.StringVar(&cfg.Trace.S3.AccessKey.Value, util.PrefixConfig(prefix, "trace.s3.access_key"), "", "s3 access key.")
-	f.StringVar(&cfg.Trace.S3.SecretKey.Value, util.PrefixConfig(prefix, "trace.s3.secret_key"), "", "s3 secret key.")
+	f.StringVar(&cfg.Trace.S3.AccessKey, util.PrefixConfig(prefix, "trace.s3.access_key"), "", "s3 access key.")
+	f.Var(&cfg.Trace.S3.SecretKey, util.PrefixConfig(prefix, "trace.s3.secret_key"), "s3 secret key.")
 	cfg.Trace.S3.HedgeRequestsUpTo = 2
 
 	cfg.Trace.GCS = &gcs.Config{}

diff --git a/pkg/cache/redis_client.go b/pkg/cache/redis_client.go
@@ -55,7 +55,7 @@ func NewRedisClient(cfg *RedisConfig) *RedisClient {
 	opt := &redis.UniversalOptions{
 		Addrs:       strings.Split(cfg.Endpoint, ","),
 		MasterName:  cfg.MasterName,
-		Password:    cfg.Password.Value,
+		Password:    cfg.Password.String(),
 		DB:          cfg.DB,
 		PoolSize:    cfg.PoolSize,
 		IdleTimeout: cfg.IdleTimeout,

diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
@@ -12,6 +12,7 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/grpcclient"
 	"github.com/grafana/dskit/services"
+	"github.com/grafana/dskit/tenant"
 	otgrpc "github.com/opentracing-contrib/go-grpc"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
@@ -25,7 +26,6 @@ import (
 	"github.com/grafana/tempo/modules/frontend/v2/frontendv2pb"
 	"github.com/grafana/tempo/pkg/scheduler/queue"
 	"github.com/grafana/tempo/pkg/scheduler/schedulerpb"
-	"github.com/grafana/tempo/pkg/tenant"
 	"github.com/grafana/tempo/pkg/util"
 	"github.com/grafana/tempo/pkg/util/httpgrpcutil"
 	"github.com/grafana/tempo/pkg/validation"

diff --git a/pkg/util/log/wrappers.go b/pkg/util/log/wrappers.go
@@ -6,7 +6,7 @@ import (
 	kitlog "github.com/go-kit/log"
 	"github.com/weaveworks/common/tracing"
 
-	"github.com/grafana/tempo/pkg/tenant"
+	"github.com/grafana/dskit/tenant"
 )
 
 // WithUserID returns a Logger that has information about the current user in

diff --git a/pkg/util/spanlogger/spanlogger.go b/pkg/util/spanlogger/spanlogger.go
@@ -9,7 +9,7 @@ import (
 	"github.com/opentracing/opentracing-go/ext"
 	otlog "github.com/opentracing/opentracing-go/log"
 
-	"github.com/grafana/tempo/pkg/tenant"
+	"github.com/grafana/dskit/tenant"
 	util_log "github.com/grafana/tempo/pkg/util/log"
 )
 

diff --git a/tempodb/backend/azure/azure_helpers.go b/tempodb/backend/azure/azure_helpers.go
@@ -21,7 +21,7 @@ const (
 )
 
 func GetContainerURL(ctx context.Context, cfg *Config, hedge bool) (blob.ContainerURL, error) {
-	accountName := cfg.StorageAccountName.String()
+	accountName := cfg.StorageAccountName
 	accountKey := cfg.StorageAccountKey.String()
 
 	if accountName == "" {

diff --git a/tempodb/backend/azure/azure_test.go b/tempodb/backend/azure/azure_test.go
@@ -66,8 +66,8 @@ func TestHedge(t *testing.T) {
 			server := fakeServer(t, tc.returnIn, &count)
 
 			r, w, _, err := New(&Config{
-				StorageAccountName: flagext.Secret{Value: "testing"},
-				StorageAccountKey:  flagext.Secret{Value: "YQo="},
+				StorageAccountName: "testing",
+				StorageAccountKey:  flagext.SecretWithValue("YQo="),
 				MaxBuffers:         3,
 				BufferSize:         1000,
 				ContainerName:      "blerg",

diff --git a/tempodb/backend/azure/config.go b/tempodb/backend/azure/config.go
@@ -7,7 +7,7 @@ import (
 )
 
 type Config struct {
-	StorageAccountName flagext.Secret `yaml:"storage-account-name"`
+	StorageAccountName string         `yaml:"storage-account-name"`
 	StorageAccountKey  flagext.Secret `yaml:"storage-account-key"`
 	ContainerName      string         `yaml:"container-name"`
 	Endpoint           string         `yaml:"endpoint-suffix"`

diff --git a/tempodb/backend/s3/config.go b/tempodb/backend/s3/config.go
@@ -10,7 +10,7 @@ type Config struct {
 	Bucket            string         `yaml:"bucket"`
 	Endpoint          string         `yaml:"endpoint"`
 	Region            string         `yaml:"region"`
-	AccessKey         flagext.Secret `yaml:"access_key"`
+	AccessKey         string         `yaml:"access_key"`
 	SecretKey         flagext.Secret `yaml:"secret_key"`
 	Insecure          bool           `yaml:"insecure"`
 	PartSize          uint64         `yaml:"part_size"`

diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go
@@ -324,7 +324,7 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) {
 		wrapCredentialsProvider(&credentials.EnvAWS{}),
 		wrapCredentialsProvider(&credentials.Static{
 			Value: credentials.Value{
-				AccessKeyID:     cfg.AccessKey.String(),
+				AccessKeyID:     cfg.AccessKey,
 				SecretAccessKey: cfg.SecretKey.String(),
 			},
 		}),

diff --git a/tempodb/backend/s3/s3_test.go b/tempodb/backend/s3/s3_test.go
@@ -49,8 +49,8 @@ func TestHedge(t *testing.T) {
 
 			r, w, _, err := New(&Config{
 				Region:            "blerg",
-				AccessKey:         flagext.Secret{Value: "test"},
-				SecretKey:         flagext.Secret{Value: "test"},
+				AccessKey:         "test",
+				SecretKey:         flagext.SecretWithValue("test"),
 				Bucket:            "blerg",
 				Insecure:          true,
 				Endpoint:          server.URL[7:], // [7:] -> strip http://

diff --git a/vendor/github.com/grafana/dskit/backoff/backoff.go b/vendor/github.com/grafana/dskit/backoff/backoff.go
diff --git a/vendor/github.com/grafana/dskit/crypto/tls/tls.go b/vendor/github.com/grafana/dskit/crypto/tls/tls.go
diff --git a/vendor/github.com/grafana/dskit/flagext/secret.go b/vendor/github.com/grafana/dskit/flagext/secret.go
diff --git a/vendor/github.com/grafana/dskit/grpcclient/grpcclient.go b/vendor/github.com/grafana/dskit/grpcclient/grpcclient.go