diff --git a/.semgrep/rpc_endpoint.yml b/.semgrep/rpc_endpoint.yml index 5f3e4d0a9bc..2446b886964 100644 --- a/.semgrep/rpc_endpoint.yml +++ b/.semgrep/rpc_endpoint.yml @@ -108,6 +108,7 @@ rules: - pattern-not: '"Status.Leader"' - pattern-not: '"Status.Peers"' - pattern-not: '"Status.Version"' + - pattern-not: '"Keyring.ListPublic"' message: "RPC method $METHOD appears to be unauthenticated" languages: - "go" diff --git a/command/agent/http.go b/command/agent/http.go index 4597496919c..d2838e7fae8 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -501,6 +501,9 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.Handle("/v1/vars", wrapCORS(s.wrap(s.VariablesListRequest))) s.mux.Handle("/v1/var/", wrapCORSWithAllowedMethods(s.wrap(s.VariableSpecificRequest), "HEAD", "GET", "PUT", "DELETE")) + // JWKS Handler + s.mux.HandleFunc("/.well-known/jwks.json", s.wrap(s.JWKSRequest)) + agentConfig := s.agent.GetConfig() uiConfigEnabled := agentConfig.UI != nil && agentConfig.UI.Enabled diff --git a/command/agent/keyring_endpoint.go b/command/agent/keyring_endpoint.go index b95b82047d1..67464af038e 100644 --- a/command/agent/keyring_endpoint.go +++ b/command/agent/keyring_endpoint.go @@ -4,12 +4,79 @@ package agent import ( + "crypto/ed25519" + "fmt" "net/http" "strings" + "time" + "github.com/go-jose/go-jose/v3" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" ) +// jwksMinMaxAge is the minimum amount of time the JWKS endpoint will instruct +// consumers to cache a response for. +const jwksMinMaxAge = 15 * time.Minute + +// JWKSRequest is used to handle JWKS requests. JWKS stands for JSON Web Key +// Sets and returns the public keys used for signing workload identities. Third +// parties may use this endpoint to validate workload identities. Consumers +// should cache this endpoint, preferably until an unknown kid is encountered. +func (s *HTTPServer) JWKSRequest(resp http.ResponseWriter, req *http.Request) (any, error) { + if req.Method != http.MethodGet { + return nil, CodedError(405, ErrInvalidMethod) + } + + args := structs.GenericRequest{} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var rpcReply structs.KeyringListPublicResponse + if err := s.agent.RPC("Keyring.ListPublic", &args, &rpcReply); err != nil { + return nil, err + } + setMeta(resp, &rpcReply.QueryMeta) + + // Key set will change after max(CreateTime) + RotationThreshold. + var newestKey int64 + jwks := make([]jose.JSONWebKey, 0, len(rpcReply.PublicKeys)) + for _, pubKey := range rpcReply.PublicKeys { + if pubKey.CreateTime > newestKey { + newestKey = pubKey.CreateTime + } + + jwk := jose.JSONWebKey{ + KeyID: pubKey.KeyID, + Algorithm: pubKey.Algorithm, + Use: pubKey.Use, + } + switch alg := pubKey.Algorithm; alg { + case structs.PubKeyAlgEdDSA: + // Convert public key bytes to an ed25519 public key + jwk.Key = ed25519.PublicKey(pubKey.PublicKey) + default: + s.logger.Warn("unknown public key algorithm. server is likely newer than client", "alg", alg) + } + + jwks = append(jwks, jwk) + } + + // Have nonzero create times and threshold so set a reasonable cache time. + if newestKey > 0 && rpcReply.RotationThreshold > 0 { + exp := time.Unix(0, newestKey).Add(rpcReply.RotationThreshold) + maxAge := helper.ExpiryToRenewTime(exp, time.Now, jwksMinMaxAge) + resp.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", int(maxAge.Seconds()))) + } + + out := &jose.JSONWebKeySet{ + Keys: jwks, + } + + return out, nil +} + // KeyringRequest is used route operator/raft API requests to the implementing // functions. func (s *HTTPServer) KeyringRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { diff --git a/go.mod b/go.mod index f64be14103a..57f4fa10556 100644 --- a/go.mod +++ b/go.mod @@ -189,6 +189,7 @@ require ( github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/fatih/color v1.15.0 github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index 87d2e2ea652..4131770bfd2 100644 --- a/go.sum +++ b/go.sum @@ -599,6 +599,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -1464,6 +1466,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/helper/retry.go b/helper/retry.go new file mode 100644 index 00000000000..726b379b39f --- /dev/null +++ b/helper/retry.go @@ -0,0 +1,27 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package helper + +import ( + "time" +) + +// ExpiryToRenewTime calculates how long until clients should try to renew +// credentials based on their expiration time and now. +// +// Renewals will begin halfway between now and the expiry plus some jitter. +// +// If the expiration is in the past or less than the min wait, then the min +// wait time will be used with jitter. +func ExpiryToRenewTime(exp time.Time, now func() time.Time, minWait time.Duration) time.Duration { + left := exp.Sub(now()) + + renewAt := left / 2 + + if renewAt < minWait { + return minWait + RandomStagger(minWait/10) + } + + return renewAt + RandomStagger(renewAt/10) +} diff --git a/helper/retry_test.go b/helper/retry_test.go new file mode 100644 index 00000000000..40f9e59d058 --- /dev/null +++ b/helper/retry_test.go @@ -0,0 +1,76 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package helper + +import ( + "testing" + "time" + + "github.com/shoenig/test/must" +) + +// TestExpiryToRenewTime_0Min asserts that ExpiryToRenewTime with a 0 min wait +// will cause an immediate renewal +func TestExpiryToRenewTime_0Min(t *testing.T) { + exp := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + now := func() time.Time { + return time.Date(2023, 1, 1, 0, 0, 1, 0, time.UTC) + } + + renew := ExpiryToRenewTime(exp, now, 0) + + must.Zero(t, renew) +} + +// TestExpiryToRenewTime_14Days asserts that ExpiryToRenewTime begins trying to +// renew at or after 7 days of a 14 day expiration window. +func TestExpiryToRenewTime_30Days(t *testing.T) { + exp := time.Date(2023, 1, 15, 0, 0, 0, 0, time.UTC) + now := func() time.Time { + return time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + } + min := 20 * time.Minute + + renew := ExpiryToRenewTime(exp, now, min) + + // Renew should be much greater than min wait + must.Greater(t, min, renew) + + // Renew should be >= 7 days + must.GreaterEq(t, 7*24*time.Hour, renew) +} + +// TestExpiryToRenewTime_UnderMin asserts that ExpiryToRenewTime uses the min +// wait + jitter if it is greater than the time until expiry. +func TestExpiryToRenewTime_UnderMin(t *testing.T) { + exp := time.Date(2023, 1, 1, 0, 0, 10, 0, time.UTC) + now := func() time.Time { + return time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + } + min := 20 * time.Second + + renew := ExpiryToRenewTime(exp, now, min) + + // Renew should be >= min wait (jitter can be 0) + must.GreaterEq(t, min, renew) + + // When we fallback to the min wait it means we miss the expiration, but this + // is necessary to prevent stampedes after outages and partitions. + must.GreaterEq(t, exp.Sub(now()), renew) +} + +// TestExpiryToRenewTime_Expired asserts that ExpiryToRenewTime defaults to +// minWait (+jitter) if the renew time has already elapsed. +func TestExpiryToRenewTime_Expired(t *testing.T) { + exp := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + now := func() time.Time { + return time.Date(2023, 2, 1, 0, 0, 0, 0, time.UTC) + } + min := time.Hour + + renew := ExpiryToRenewTime(exp, now, min) + + must.Greater(t, min, renew) + must.Less(t, min*2, renew) +} diff --git a/nomad/encrypter.go b/nomad/encrypter.go index e84bb649f7a..c7c071386d3 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -392,6 +392,26 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { }, nil } +// GetPublicKey returns the public signing key for the requested key id or an +// error if the key could not be found. +func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error) { + e.lock.Lock() + defer e.lock.Unlock() + + ks, err := e.keysetByIDLocked(keyID) + if err != nil { + return nil, err + } + + return &structs.KeyringPublicKey{ + KeyID: ks.rootKey.Meta.KeyID, + PublicKey: ks.privateKey.Public().(ed25519.PublicKey), + Algorithm: structs.PubKeyAlgEdDSA, + Use: structs.PubKeyUseSig, + CreateTime: ks.rootKey.Meta.CreateTime, + }, nil +} + // newKMSWrapper returns a go-kms-wrapping interface the caller can use to // encrypt the RootKey with a key encryption key (KEK). This is a bit of // security theatre for local on-disk key material, but gives us a shim for diff --git a/nomad/keyring_endpoint.go b/nomad/keyring_endpoint.go index f921d0cbfb6..8b6b5d80bb5 100644 --- a/nomad/keyring_endpoint.go +++ b/nomad/keyring_endpoint.go @@ -360,3 +360,66 @@ func (k *Keyring) Delete(args *structs.KeyringDeleteRootKeyRequest, reply *struc reply.Index = index return nil } + +// ListPublic signing keys used for workload identities. This RPC is used to +// back a JWKS endpoint. +// +// Unauthenticated because public keys are not sensitive. +func (k *Keyring) ListPublic(args *structs.GenericRequest, reply *structs.KeyringListPublicResponse) error { + + // JWKS is a public endpoint: intentionally ignore auth errors and only + // authenticate to measure rate metrics. + k.srv.Authenticate(k.ctx, args) + if done, err := k.srv.forward("Keyring.ListPublic", args, args, reply); done { + return err + } + k.srv.MeasureRPCRate("keyring", structs.RateMetricList, args) + + defer metrics.MeasureSince([]string{"nomad", "keyring", "list_public"}, time.Now()) + + // Expose root_key_rotation_threshold so consumers can determine reasonable + // cache settings. + reply.RotationThreshold = k.srv.config.RootKeyRotationThreshold + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, s *state.StateStore) error { + + // retrieve all the key metadata + snap, err := k.srv.fsm.State().Snapshot() + if err != nil { + return err + } + iter, err := snap.RootKeyMetas(ws) + if err != nil { + return err + } + + pubKeys := []*structs.KeyringPublicKey{} + for { + raw := iter.Next() + if raw == nil { + break + } + + keyMeta := raw.(*structs.RootKeyMeta) + if keyMeta.State == structs.RootKeyStateDeprecated { + // Only include valid keys + continue + } + + pubKey, err := k.encrypter.GetPublicKey(keyMeta.KeyID) + if err != nil { + return err + } + + pubKeys = append(pubKeys, pubKey) + } + reply.PublicKeys = pubKeys + return k.srv.replySetIndex(state.TableRootKeyMeta, &reply.QueryMeta) + }, + } + return k.srv.blockingRPC(&opts) +} diff --git a/nomad/keyring_endpoint_test.go b/nomad/keyring_endpoint_test.go index 4fe0df70190..fc295359223 100644 --- a/nomad/keyring_endpoint_test.go +++ b/nomad/keyring_endpoint_test.go @@ -9,6 +9,7 @@ import ( "time" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/shoenig/test/must" "github.com/stretchr/testify/require" "github.com/hashicorp/nomad/ci" @@ -295,3 +296,59 @@ func TestKeyringEndpoint_Rotate(t *testing.T) { gotKey := getResp.Key require.Len(t, gotKey.Key, 32) } + +// TestKeyringEndpoint_ListPublic asserts the Keyring.ListPublic RPC returns +// all keys which may be in use for active crytpographic material (variables, +// valid JWTs). +func TestKeyringEndpoint_ListPublic(t *testing.T) { + + ci.Parallel(t) + srv, rootToken, shutdown := TestACLServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer shutdown() + testutil.WaitForLeader(t, srv.RPC) + codec := rpcClient(t, srv) + + // Assert 1 key exists and normal fields are set + req := structs.GenericRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + AuthToken: "ignored!", + }, + } + var resp structs.KeyringListPublicResponse + must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.ListPublic", &req, &resp)) + must.Eq(t, srv.config.RootKeyRotationThreshold, resp.RotationThreshold) + must.Len(t, 1, resp.PublicKeys) + must.NonZero(t, resp.Index) + + // Rotate keys and assert there are now 2 keys + rotateReq := &structs.KeyringRotateRootKeyRequest{ + WriteRequest: structs.WriteRequest{ + Region: "global", + AuthToken: rootToken.SecretID, + }, + } + var rotateResp structs.KeyringRotateRootKeyResponse + must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.Rotate", rotateReq, &rotateResp)) + must.NotEq(t, resp.Index, rotateResp.Index) + + // Verify we have a new key and the old one is inactive + var resp2 structs.KeyringListPublicResponse + must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.ListPublic", &req, &resp2)) + must.Eq(t, srv.config.RootKeyRotationThreshold, resp2.RotationThreshold) + must.Len(t, 2, resp2.PublicKeys) + must.NonZero(t, resp2.Index) + + found := false + for _, pk := range resp2.PublicKeys { + if pk.KeyID == resp.PublicKeys[0].KeyID { + must.False(t, found, must.Sprint("found the original public key twice")) + found = true + must.Eq(t, resp.PublicKeys[0], pk) + break + } + } + must.True(t, found, must.Sprint("original public key missing after rotation")) +} diff --git a/nomad/structs/keyring.go b/nomad/structs/keyring.go index 5f41dad2d60..73c34fcfbd7 100644 --- a/nomad/structs/keyring.go +++ b/nomad/structs/keyring.go @@ -12,6 +12,16 @@ import ( "github.com/hashicorp/nomad/helper/uuid" ) +const ( + // PubKeyAlgEdDSA is the JWA (JSON Web Algorithm) for ed25519 public keys + // used for signatures. + PubKeyAlgEdDSA = "EdDSA" + + // PubKeyUseSig is the JWK (JSON Web Key) "use" parameter value for + // signatures. + PubKeyUseSig = "sig" +) + // RootKey is used to encrypt and decrypt variables. It is never stored in raft. type RootKey struct { Meta *RootKeyMeta @@ -243,3 +253,38 @@ type KeyringDeleteRootKeyRequest struct { type KeyringDeleteRootKeyResponse struct { WriteMeta } + +// KeyringListPublicResponse lists public key components of signing keys. Used +// to build a JWKS endpoint. +type KeyringListPublicResponse struct { + PublicKeys []*KeyringPublicKey + + // RotationThreshold exposes root_key_rotation_threshold so that HTTP + // endpoints may set a reasonable cache control header informing consumers + // when to expect a new key. + RotationThreshold time.Duration + + QueryMeta +} + +// KeyringPublicKey is the public key component of a signing key. Used to build +// a JWKS endpoint. +type KeyringPublicKey struct { + KeyID string + PublicKey []byte + + // Algorithm should be the JWT "alg" parameter. So "EdDSA" for Ed25519 public + // keys used to validate signatures. + Algorithm string + + // Use should be the JWK "use" parameter as defined in + // https://datatracker.ietf.org/doc/html/rfc7517#section-4.2. + // + // "sig" and "enc" being the two standard values with "sig" being the use for + // workload identity JWT signing. + Use string + + // CreateTime + root_key_rotation_threshold = when consumers should look for + // a new key. Therefore this field can be used for cache control. + CreateTime int64 +}