Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PWX-36873: Add vault cooldowns #86

Merged
merged 8 commits into from
Apr 16, 2024
Merged

PWX-36873: Add vault cooldowns #86

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
cce9915
PWX-36873: vault cooldowns
zoxpx Apr 13, 2024
f9fde71
adding UT
zoxpx Apr 15, 2024
527629b
fixing UT
zoxpx Apr 15, 2024
7c3f770
fixing UT r2
zoxpx Apr 15, 2024
5c9968f
fixing UT r3
zoxpx Apr 15, 2024
3aaff8a
fixing UT r4
zoxpx Apr 15, 2024
148196b
review feedback
zoxpx Apr 15, 2024
575946c
Revert "fixing UT r4"
zoxpx Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 92 additions & 17 deletions vault/vault.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package vault

import (
"errors"
"fmt"
"path"
"strings"
"sync"
"time"

"github.com/hashicorp/vault/api"
"github.com/libopenstorage/secrets"
Expand All @@ -13,15 +15,17 @@ import (
)

const (
Name = secrets.TypeVault
DefaultBackendPath = "secret/"
VaultBackendPathKey = "VAULT_BACKEND_PATH"
VaultBackendKey = "VAULT_BACKEND"
kvVersionKey = "version"
kvDataKey = "data"
kvMetadataKey = "metadata"
kvVersion1 = "kv"
kvVersion2 = "kv-v2"
Name = secrets.TypeVault
DefaultBackendPath = "secret/"
VaultBackendPathKey = "VAULT_BACKEND_PATH"
VaultBackendKey = "VAULT_BACKEND"
VaultCooldownPeriod = "VAULT_COOLDOWN_PERIOD"
kvVersionKey = "version"
kvDataKey = "data"
kvMetadataKey = "metadata"
kvVersion1 = "kv"
kvVersion2 = "kv-v2"
defaultCooldownPeriod = 5 * time.Minute

AuthMethodKubernetes = utils.AuthMethodKubernetes
AuthMethod = utils.AuthMethod
Expand Down Expand Up @@ -52,12 +56,15 @@ type vaultSecrets struct {
isKvBackendV2 bool
autoAuth bool
config map[string]interface{}
cooldown time.Time
}

// These variables are helpful in testing to stub method call from packages
var (
newVaultClient = api.NewClient
isKvV2 = isKvBackendV2
newVaultClient = api.NewClient
isKvV2 = isKvBackendV2
errInCooldown = errors.New("vault client is in cooldown")
zoxpx marked this conversation as resolved.
Show resolved Hide resolved
confCooldownPeriod time.Duration
)

func New(
Expand Down Expand Up @@ -109,7 +116,7 @@ func New(
authMethod = method
}

logrus.Infof("Authenticated to Vault with %v\n", authMethod)
logrus.Infof("Will authenticate to Vault via %v", authMethod)

backendPath := utils.GetVaultParam(secretConfig, VaultBackendPathKey)
if backendPath == "" {
Expand All @@ -130,6 +137,21 @@ func New(
return nil, err
}
}

if cd := utils.GetVaultParam(secretConfig, VaultCooldownPeriod); cd != "" {
confCooldownPeriod, err = time.ParseDuration(cd)
if err == nil && confCooldownPeriod > time.Minute {
logrus.Infof("cooldown period is set to %s via %s=%s", confCooldownPeriod, VaultCooldownPeriod, cd)
} else {
// let's turn OFF the cooldown, if one passed invalid value (or "disabled")
logrus.WithError(err).Warnf("cooldown period is invalid (%s=%s) -- cooldowns turned OFF", VaultCooldownPeriod, cd)
confCooldownPeriod = 0
}
zoxpx marked this conversation as resolved.
Show resolved Hide resolved
} else {
confCooldownPeriod = defaultCooldownPeriod
logrus.Infof("cooldown period is set to %s", confCooldownPeriod)
}

return &vaultSecrets{
endpoint: config.Address,
namespace: namespace,
Expand Down Expand Up @@ -261,6 +283,9 @@ func (v *vaultSecrets) ListSecrets() ([]string, error) {
}

func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
if v.isInCooldown() {
return nil, errInCooldown
}
if v.autoAuth {
v.lockClientToken.Lock()
defer v.lockClientToken.Unlock()
Expand All @@ -272,7 +297,7 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {

secretValue, err := v.lockedRead(path.Path())
if v.isTokenExpired(err) {
if err = v.renewToken(path.Namespace()); err != nil {
if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
return nil, fmt.Errorf("failed to renew token: %s", err)
}
return v.lockedRead(path.Path())
Expand All @@ -281,6 +306,9 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
}

func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Secret, error) {
if v.isInCooldown() {
return nil, errInCooldown
}
if v.autoAuth {
v.lockClientToken.Lock()
defer v.lockClientToken.Unlock()
Expand All @@ -292,7 +320,7 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se

secretValue, err := v.lockedWrite(path.Path(), data)
if v.isTokenExpired(err) {
if err = v.renewToken(path.Namespace()); err != nil {
if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
return nil, fmt.Errorf("failed to renew token: %s", err)
}
return v.lockedWrite(path.Path(), data)
Expand All @@ -301,6 +329,9 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se
}

func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {
if v.isInCooldown() {
return nil, errInCooldown
}
if v.autoAuth {
v.lockClientToken.Lock()
defer v.lockClientToken.Unlock()
Expand All @@ -312,7 +343,7 @@ func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {

secretValue, err := v.lockedDelete(path.Path())
if v.isTokenExpired(err) {
if err = v.renewToken(path.Namespace()); err != nil {
if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
return nil, fmt.Errorf("failed to renew token: %s", err)
}
return v.lockedDelete(path.Path())
Expand Down Expand Up @@ -359,6 +390,50 @@ func (v *vaultSecrets) renewToken(namespace string) error {
return nil
}

func (v *vaultSecrets) renewTokenWithCooldown(namespace string) error {
if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
return v.renewToken(namespace)
} else if v.isInCooldown() {
return errInCooldown
}

err := v.renewToken(namespace)
if v.isTokenExpired(err) {
v.setCooldown(confCooldownPeriod)
} else if err == nil {
v.setCooldown(0) // clear cooldown
}
return err
}

func (v *vaultSecrets) isInCooldown() bool {
if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
return false
}
v.mu.RLock()
defer v.mu.RUnlock()
if v.cooldown.IsZero() {
return false
}
return time.Now().Before(v.cooldown)
}

func (v *vaultSecrets) setCooldown(dur time.Duration) {
if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
return
}
v.mu.Lock()
defer v.mu.Unlock()
if dur > 0 {
v.cooldown = time.Now().Add(dur)
logrus.WithField("nextRetryAt", v.cooldown.Round(100*time.Millisecond)).
Warnf("putting vault client in cooldown for %s", confCooldownPeriod)
} else {
logrus.Debug("clearing vault client cooldown")
v.cooldown = time.Time{}
}
}

func (v *vaultSecrets) isTokenExpired(err error) bool {
return err != nil && v.autoAuth && strings.Contains(err.Error(), "permission denied")
}
Expand All @@ -373,7 +448,7 @@ func (v *vaultSecrets) setNamespaceToken(namespace string) error {
return nil
}

return v.renewToken(namespace)
return v.renewTokenWithCooldown(namespace)
}

func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
Expand All @@ -393,7 +468,7 @@ func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
}
}

return false, fmt.Errorf("Secrets engine with mount path '%s' not found",
return false, fmt.Errorf("secrets engine with mount path '%s' not found",
backendPath)
}

Expand Down
Loading
Loading