PWX-36873: Add vault cooldowns

vault/utils/utils.go

@@ -49,6 +49,7 @@ var (
 
 	ErrAuthMethodUnknown = errors.New("unknown auth method")
 	ErrKubernetesRole    = errors.New(AuthKubernetesRole + " not set")
+	ErrInCooldown        = errors.New("vault client is in cooldown")
 )
 
 // IsValidAddr checks address has the correct format.

vault/vault.go

@@ -5,6 +5,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/hashicorp/vault/api"
 	"github.com/libopenstorage/secrets"
@@ -13,15 +14,17 @@ import (
 )
 
 const (
-	Name                = secrets.TypeVault
-	DefaultBackendPath  = "secret/"
-	VaultBackendPathKey = "VAULT_BACKEND_PATH"
-	VaultBackendKey     = "VAULT_BACKEND"
-	kvVersionKey        = "version"
-	kvDataKey           = "data"
-	kvMetadataKey       = "metadata"
-	kvVersion1          = "kv"
-	kvVersion2          = "kv-v2"
+	Name                  = secrets.TypeVault
+	DefaultBackendPath    = "secret/"
+	VaultBackendPathKey   = "VAULT_BACKEND_PATH"
+	VaultBackendKey       = "VAULT_BACKEND"
+	VaultCooldownPeriod   = "VAULT_COOLDOWN_PERIOD"
+	kvVersionKey          = "version"
+	kvDataKey             = "data"
+	kvMetadataKey         = "metadata"
+	kvVersion1            = "kv"
+	kvVersion2            = "kv-v2"
+	defaultCooldownPeriod = 5 * time.Minute
 
 	AuthMethodKubernetes    = utils.AuthMethodKubernetes
 	AuthMethod              = utils.AuthMethod
@@ -52,12 +55,14 @@ type vaultSecrets struct {
 	isKvBackendV2 bool
 	autoAuth      bool
 	config        map[string]interface{}
+	cooldown      time.Time
 }
 
 // These variables are helpful in testing to stub method call from packages
 var (
-	newVaultClient = api.NewClient
-	isKvV2         = isKvBackendV2
+	newVaultClient     = api.NewClient
+	isKvV2             = isKvBackendV2
+	confCooldownPeriod time.Duration
 )
 
 func New(
@@ -109,7 +114,7 @@ func New(
 		authMethod = method
 	}
 
-	logrus.Infof("Authenticated to Vault with %v\n", authMethod)
+	logrus.Infof("Will authenticate to Vault via %v", authMethod)
 
 	backendPath := utils.GetVaultParam(secretConfig, VaultBackendPathKey)
 	if backendPath == "" {
@@ -130,6 +135,20 @@ func New(
 			return nil, err
 		}
 	}
+
+	confCooldownPeriod = defaultCooldownPeriod
+	if cd := utils.GetVaultParam(secretConfig, VaultCooldownPeriod); cd != "" {
+		if cd == "0" {
+			logrus.Warnf("cooldown period is disabled via %s=%s", VaultCooldownPeriod, cd)
+			confCooldownPeriod = 0
+		} else if confCooldownPeriod, err = time.ParseDuration(cd); err == nil && confCooldownPeriod > time.Minute {
+			logrus.Infof("cooldown period is set to %s", confCooldownPeriod)
+		} else {
+			return nil, fmt.Errorf("invalid cooldown period: %s=%s", VaultCooldownPeriod, cd)
+		}
+	}
+	logrus.Infof("cooldown period is set to %s", confCooldownPeriod)
+
 	return &vaultSecrets{
 		endpoint:         config.Address,
 		namespace:        namespace,
@@ -261,6 +280,9 @@ func (v *vaultSecrets) ListSecrets() ([]string, error) {
 }
 
 func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
+	if v.isInCooldown() {
+		return nil, utils.ErrInCooldown
+	}
 	if v.autoAuth {
 		v.lockClientToken.Lock()
 		defer v.lockClientToken.Unlock()
@@ -272,7 +294,7 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
 
 	secretValue, err := v.lockedRead(path.Path())
 	if v.isTokenExpired(err) {
-		if err = v.renewToken(path.Namespace()); err != nil {
+		if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
 			return nil, fmt.Errorf("failed to renew token: %s", err)
 		}
 		return v.lockedRead(path.Path())
@@ -281,6 +303,9 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
 }
 
 func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Secret, error) {
+	if v.isInCooldown() {
+		return nil, utils.ErrInCooldown
+	}
 	if v.autoAuth {
 		v.lockClientToken.Lock()
 		defer v.lockClientToken.Unlock()
@@ -292,7 +317,7 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se
 
 	secretValue, err := v.lockedWrite(path.Path(), data)
 	if v.isTokenExpired(err) {
-		if err = v.renewToken(path.Namespace()); err != nil {
+		if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
 			return nil, fmt.Errorf("failed to renew token: %s", err)
 		}
 		return v.lockedWrite(path.Path(), data)
@@ -301,6 +326,9 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se
 }
 
 func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {
+	if v.isInCooldown() {
+		return nil, utils.ErrInCooldown
+	}
 	if v.autoAuth {
 		v.lockClientToken.Lock()
 		defer v.lockClientToken.Unlock()
@@ -312,7 +340,7 @@ func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {
 
 	secretValue, err := v.lockedDelete(path.Path())
 	if v.isTokenExpired(err) {
-		if err = v.renewToken(path.Namespace()); err != nil {
+		if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
 			return nil, fmt.Errorf("failed to renew token: %s", err)
 		}
 		return v.lockedDelete(path.Path())
@@ -359,6 +387,50 @@ func (v *vaultSecrets) renewToken(namespace string) error {
 	return nil
 }
 
+func (v *vaultSecrets) renewTokenWithCooldown(namespace string) error {
+	if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
+		return v.renewToken(namespace)
+	} else if v.isInCooldown() {
+		return utils.ErrInCooldown
+	}
+
+	err := v.renewToken(namespace)
+	if v.isTokenExpired(err) {
+		v.setCooldown(confCooldownPeriod)
+	} else if err == nil {
+		v.setCooldown(0) // clear cooldown
+	}
+	return err
+}
+
+func (v *vaultSecrets) isInCooldown() bool {
+	if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
+		return false
+	}
+	v.mu.RLock()
+	defer v.mu.RUnlock()
+	if v.cooldown.IsZero() {
+		return false
+	}
+	return time.Now().Before(v.cooldown)
+}
+
+func (v *vaultSecrets) setCooldown(dur time.Duration) {
+	if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
+		return
+	}
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	if dur > 0 {
+		v.cooldown = time.Now().Add(dur)
+		logrus.WithField("nextRetryAt", v.cooldown.Round(100*time.Millisecond)).
+			Warnf("putting vault client in cooldown for %s", confCooldownPeriod)
+	} else {
+		logrus.Debug("clearing vault client cooldown")
+		v.cooldown = time.Time{}
+	}
+}
+
 func (v *vaultSecrets) isTokenExpired(err error) bool {
 	return err != nil && v.autoAuth && strings.Contains(err.Error(), "permission denied")
 }
@@ -373,7 +445,7 @@ func (v *vaultSecrets) setNamespaceToken(namespace string) error {
 		return nil
 	}
 
-	return v.renewToken(namespace)
+	return v.renewTokenWithCooldown(namespace)
 }
 
 func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
@@ -393,7 +465,7 @@ func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
 		}
 	}
 
-	return false, fmt.Errorf("Secrets engine with mount path '%s' not found",
+	return false, fmt.Errorf("secrets engine with mount path '%s' not found",
 		backendPath)
 }