Skip to content

Commit

Permalink
PWX-36873: Add vault cooldowns (#86)
Browse files Browse the repository at this point in the history 
* double "permission denied" REST error will put vault client into 5 minutes cooldown for all REST calls
* can disable via `VAULT_COOLDOWN_PERIOD:0`

Signed-off-by: Zoran Rajic <[email protected]>
  • Loading branch information
@zoxpx
zoxpx authored Apr 16, 2024
1 parent 5f4b25c commit a17cf7f
Show file tree
Hide file tree
Showing 3 changed files with 336 additions and 17 deletions.
1 change: 1 addition & 0 deletions vault/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ var (

ErrAuthMethodUnknown = errors.New("unknown auth method")
ErrKubernetesRole = errors.New(AuthKubernetesRole + " not set")
ErrInCooldown = errors.New("vault client is in cooldown")
)

// IsValidAddr checks address has the correct format.
Expand Down
106 changes: 89 additions & 17 deletions vault/vault.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"path"
"strings"
"sync"
"time"

"github.com/hashicorp/vault/api"
"github.com/libopenstorage/secrets"
Expand All @@ -13,15 +14,17 @@ import (
)

const (
Name = secrets.TypeVault
DefaultBackendPath = "secret/"
VaultBackendPathKey = "VAULT_BACKEND_PATH"
VaultBackendKey = "VAULT_BACKEND"
kvVersionKey = "version"
kvDataKey = "data"
kvMetadataKey = "metadata"
kvVersion1 = "kv"
kvVersion2 = "kv-v2"
Name = secrets.TypeVault
DefaultBackendPath = "secret/"
VaultBackendPathKey = "VAULT_BACKEND_PATH"
VaultBackendKey = "VAULT_BACKEND"
VaultCooldownPeriod = "VAULT_COOLDOWN_PERIOD"
kvVersionKey = "version"
kvDataKey = "data"
kvMetadataKey = "metadata"
kvVersion1 = "kv"
kvVersion2 = "kv-v2"
defaultCooldownPeriod = 5 * time.Minute

AuthMethodKubernetes = utils.AuthMethodKubernetes
AuthMethod = utils.AuthMethod
Expand Down Expand Up @@ -52,12 +55,14 @@ type vaultSecrets struct {
isKvBackendV2 bool
autoAuth bool
config map[string]interface{}
cooldown time.Time
}

// These variables are helpful in testing to stub method call from packages
var (
newVaultClient = api.NewClient
isKvV2 = isKvBackendV2
newVaultClient = api.NewClient
isKvV2 = isKvBackendV2
confCooldownPeriod time.Duration
)

func New(
Expand Down Expand Up @@ -109,7 +114,7 @@ func New(
authMethod = method
}

logrus.Infof("Authenticated to Vault with %v\n", authMethod)
logrus.Infof("Will authenticate to Vault via %v", authMethod)

backendPath := utils.GetVaultParam(secretConfig, VaultBackendPathKey)
if backendPath == "" {
Expand All @@ -130,6 +135,20 @@ func New(
return nil, err
}
}

confCooldownPeriod = defaultCooldownPeriod
if cd := utils.GetVaultParam(secretConfig, VaultCooldownPeriod); cd != "" {
if cd == "0" {
logrus.Warnf("cooldown period is disabled via %s=%s", VaultCooldownPeriod, cd)
confCooldownPeriod = 0
} else if confCooldownPeriod, err = time.ParseDuration(cd); err == nil && confCooldownPeriod > time.Minute {
logrus.Infof("cooldown period is set to %s", confCooldownPeriod)
} else {
return nil, fmt.Errorf("invalid cooldown period: %s=%s", VaultCooldownPeriod, cd)
}
}
logrus.Infof("cooldown period is set to %s", confCooldownPeriod)

return &vaultSecrets{
endpoint: config.Address,
namespace: namespace,
Expand Down Expand Up @@ -261,6 +280,9 @@ func (v *vaultSecrets) ListSecrets() ([]string, error) {
}

func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
if v.isInCooldown() {
return nil, utils.ErrInCooldown
}
if v.autoAuth {
v.lockClientToken.Lock()
defer v.lockClientToken.Unlock()
Expand All @@ -272,7 +294,7 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {

secretValue, err := v.lockedRead(path.Path())
if v.isTokenExpired(err) {
if err = v.renewToken(path.Namespace()); err != nil {
if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
return nil, fmt.Errorf("failed to renew token: %s", err)
}
return v.lockedRead(path.Path())
Expand All @@ -281,6 +303,9 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
}

func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Secret, error) {
if v.isInCooldown() {
return nil, utils.ErrInCooldown
}
if v.autoAuth {
v.lockClientToken.Lock()
defer v.lockClientToken.Unlock()
Expand All @@ -292,7 +317,7 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se

secretValue, err := v.lockedWrite(path.Path(), data)
if v.isTokenExpired(err) {
if err = v.renewToken(path.Namespace()); err != nil {
if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
return nil, fmt.Errorf("failed to renew token: %s", err)
}
return v.lockedWrite(path.Path(), data)
Expand All @@ -301,6 +326,9 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se
}

func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {
if v.isInCooldown() {
return nil, utils.ErrInCooldown
}
if v.autoAuth {
v.lockClientToken.Lock()
defer v.lockClientToken.Unlock()
Expand All @@ -312,7 +340,7 @@ func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {

secretValue, err := v.lockedDelete(path.Path())
if v.isTokenExpired(err) {
if err = v.renewToken(path.Namespace()); err != nil {
if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
return nil, fmt.Errorf("failed to renew token: %s", err)
}
return v.lockedDelete(path.Path())
Expand Down Expand Up @@ -359,6 +387,50 @@ func (v *vaultSecrets) renewToken(namespace string) error {
return nil
}

func (v *vaultSecrets) renewTokenWithCooldown(namespace string) error {
if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
return v.renewToken(namespace)
} else if v.isInCooldown() {
return utils.ErrInCooldown
}

err := v.renewToken(namespace)
if v.isTokenExpired(err) {
v.setCooldown(confCooldownPeriod)
} else if err == nil {
v.setCooldown(0) // clear cooldown
}
return err
}

func (v *vaultSecrets) isInCooldown() bool {
if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
return false
}
v.mu.RLock()
defer v.mu.RUnlock()
if v.cooldown.IsZero() {
return false
}
return time.Now().Before(v.cooldown)
}

func (v *vaultSecrets) setCooldown(dur time.Duration) {
if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
return
}
v.mu.Lock()
defer v.mu.Unlock()
if dur > 0 {
v.cooldown = time.Now().Add(dur)
logrus.WithField("nextRetryAt", v.cooldown.Round(100*time.Millisecond)).
Warnf("putting vault client in cooldown for %s", confCooldownPeriod)
} else {
logrus.Debug("clearing vault client cooldown")
v.cooldown = time.Time{}
}
}

func (v *vaultSecrets) isTokenExpired(err error) bool {
return err != nil && v.autoAuth && strings.Contains(err.Error(), "permission denied")
}
Expand All @@ -373,7 +445,7 @@ func (v *vaultSecrets) setNamespaceToken(namespace string) error {
return nil
}

return v.renewToken(namespace)
return v.renewTokenWithCooldown(namespace)
}

func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
Expand All @@ -393,7 +465,7 @@ func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
}
}

return false, fmt.Errorf("Secrets engine with mount path '%s' not found",
return false, fmt.Errorf("secrets engine with mount path '%s' not found",
backendPath)
}

Expand Down
Loading

0 comments on commit a17cf7f

Please sign in to comment.