Skip to content

Commit

Permalink
core: add jwks rpc and http api (#18035)
Browse files Browse the repository at this point in the history
Add JWKS endpoint to HTTP API for exposing the root public signing keys used for signing workload identity JWTs.

Part 1 of N components as part of making workload identities consumable by third party services such as Consul and Vault. Identity attenuation (audience) and expiration (+renewal) are necessary to securely use workload identities with 3rd parties, so this merge does not yet document this endpoint.

---------

Co-authored-by: Tim Gross <[email protected]>
  • Loading branch information
schmichael and tgross authored Jul 27, 2023
1 parent ee0b104 commit d14362e
Show file tree
Hide file tree
Showing 11 changed files with 363 additions and 0 deletions.
1 change: 1 addition & 0 deletions .semgrep/rpc_endpoint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ rules:
- pattern-not: '"Status.Leader"'
- pattern-not: '"Status.Peers"'
- pattern-not: '"Status.Version"'
- pattern-not: '"Keyring.ListPublic"'
message: "RPC method $METHOD appears to be unauthenticated"
languages:
- "go"
Expand Down
3 changes: 3 additions & 0 deletions command/agent/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,9 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
s.mux.Handle("/v1/vars", wrapCORS(s.wrap(s.VariablesListRequest)))
s.mux.Handle("/v1/var/", wrapCORSWithAllowedMethods(s.wrap(s.VariableSpecificRequest), "HEAD", "GET", "PUT", "DELETE"))

// JWKS Handler
s.mux.HandleFunc("/.well-known/jwks.json", s.wrap(s.JWKSRequest))

agentConfig := s.agent.GetConfig()
uiConfigEnabled := agentConfig.UI != nil && agentConfig.UI.Enabled

Expand Down
67 changes: 67 additions & 0 deletions command/agent/keyring_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,79 @@
package agent

import (
"crypto/ed25519"
"fmt"
"net/http"
"strings"
"time"

"github.com/go-jose/go-jose/v3"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
)

// jwksMinMaxAge is the minimum amount of time the JWKS endpoint will instruct
// consumers to cache a response for.
const jwksMinMaxAge = 15 * time.Minute

// JWKSRequest is used to handle JWKS requests. JWKS stands for JSON Web Key
// Sets and returns the public keys used for signing workload identities. Third
// parties may use this endpoint to validate workload identities. Consumers
// should cache this endpoint, preferably until an unknown kid is encountered.
func (s *HTTPServer) JWKSRequest(resp http.ResponseWriter, req *http.Request) (any, error) {
if req.Method != http.MethodGet {
return nil, CodedError(405, ErrInvalidMethod)
}

args := structs.GenericRequest{}
if s.parse(resp, req, &args.Region, &args.QueryOptions) {
return nil, nil
}

var rpcReply structs.KeyringListPublicResponse
if err := s.agent.RPC("Keyring.ListPublic", &args, &rpcReply); err != nil {
return nil, err
}
setMeta(resp, &rpcReply.QueryMeta)

// Key set will change after max(CreateTime) + RotationThreshold.
var newestKey int64
jwks := make([]jose.JSONWebKey, 0, len(rpcReply.PublicKeys))
for _, pubKey := range rpcReply.PublicKeys {
if pubKey.CreateTime > newestKey {
newestKey = pubKey.CreateTime
}

jwk := jose.JSONWebKey{
KeyID: pubKey.KeyID,
Algorithm: pubKey.Algorithm,
Use: pubKey.Use,
}
switch alg := pubKey.Algorithm; alg {
case structs.PubKeyAlgEdDSA:
// Convert public key bytes to an ed25519 public key
jwk.Key = ed25519.PublicKey(pubKey.PublicKey)
default:
s.logger.Warn("unknown public key algorithm. server is likely newer than client", "alg", alg)
}

jwks = append(jwks, jwk)
}

// Have nonzero create times and threshold so set a reasonable cache time.
if newestKey > 0 && rpcReply.RotationThreshold > 0 {
exp := time.Unix(0, newestKey).Add(rpcReply.RotationThreshold)
maxAge := helper.ExpiryToRenewTime(exp, time.Now, jwksMinMaxAge)
resp.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", int(maxAge.Seconds())))
}

out := &jose.JSONWebKeySet{
Keys: jwks,
}

return out, nil
}

// KeyringRequest is used route operator/raft API requests to the implementing
// functions.
func (s *HTTPServer) KeyringRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ require (
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/fatih/color v1.15.0
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/go-jose/go-jose/v3 v3.0.0
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
Expand Down Expand Up @@ -1464,6 +1466,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
Expand Down
27 changes: 27 additions & 0 deletions helper/retry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package helper

import (
"time"
)

// ExpiryToRenewTime calculates how long until clients should try to renew
// credentials based on their expiration time and now.
//
// Renewals will begin halfway between now and the expiry plus some jitter.
//
// If the expiration is in the past or less than the min wait, then the min
// wait time will be used with jitter.
func ExpiryToRenewTime(exp time.Time, now func() time.Time, minWait time.Duration) time.Duration {
left := exp.Sub(now())

renewAt := left / 2

if renewAt < minWait {
return minWait + RandomStagger(minWait/10)
}

return renewAt + RandomStagger(renewAt/10)
}
76 changes: 76 additions & 0 deletions helper/retry_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package helper

import (
"testing"
"time"

"github.com/shoenig/test/must"
)

// TestExpiryToRenewTime_0Min asserts that ExpiryToRenewTime with a 0 min wait
// will cause an immediate renewal
func TestExpiryToRenewTime_0Min(t *testing.T) {
exp := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
now := func() time.Time {
return time.Date(2023, 1, 1, 0, 0, 1, 0, time.UTC)
}

renew := ExpiryToRenewTime(exp, now, 0)

must.Zero(t, renew)
}

// TestExpiryToRenewTime_14Days asserts that ExpiryToRenewTime begins trying to
// renew at or after 7 days of a 14 day expiration window.
func TestExpiryToRenewTime_30Days(t *testing.T) {
exp := time.Date(2023, 1, 15, 0, 0, 0, 0, time.UTC)
now := func() time.Time {
return time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
}
min := 20 * time.Minute

renew := ExpiryToRenewTime(exp, now, min)

// Renew should be much greater than min wait
must.Greater(t, min, renew)

// Renew should be >= 7 days
must.GreaterEq(t, 7*24*time.Hour, renew)
}

// TestExpiryToRenewTime_UnderMin asserts that ExpiryToRenewTime uses the min
// wait + jitter if it is greater than the time until expiry.
func TestExpiryToRenewTime_UnderMin(t *testing.T) {
exp := time.Date(2023, 1, 1, 0, 0, 10, 0, time.UTC)
now := func() time.Time {
return time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
}
min := 20 * time.Second

renew := ExpiryToRenewTime(exp, now, min)

// Renew should be >= min wait (jitter can be 0)
must.GreaterEq(t, min, renew)

// When we fallback to the min wait it means we miss the expiration, but this
// is necessary to prevent stampedes after outages and partitions.
must.GreaterEq(t, exp.Sub(now()), renew)
}

// TestExpiryToRenewTime_Expired asserts that ExpiryToRenewTime defaults to
// minWait (+jitter) if the renew time has already elapsed.
func TestExpiryToRenewTime_Expired(t *testing.T) {
exp := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
now := func() time.Time {
return time.Date(2023, 2, 1, 0, 0, 0, 0, time.UTC)
}
min := time.Hour

renew := ExpiryToRenewTime(exp, now, min)

must.Greater(t, min, renew)
must.Less(t, min*2, renew)
}
20 changes: 20 additions & 0 deletions nomad/encrypter.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,26 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) {
}, nil
}

// GetPublicKey returns the public signing key for the requested key id or an
// error if the key could not be found.
func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error) {
e.lock.Lock()
defer e.lock.Unlock()

ks, err := e.keysetByIDLocked(keyID)
if err != nil {
return nil, err
}

return &structs.KeyringPublicKey{
KeyID: ks.rootKey.Meta.KeyID,
PublicKey: ks.privateKey.Public().(ed25519.PublicKey),
Algorithm: structs.PubKeyAlgEdDSA,
Use: structs.PubKeyUseSig,
CreateTime: ks.rootKey.Meta.CreateTime,
}, nil
}

// newKMSWrapper returns a go-kms-wrapping interface the caller can use to
// encrypt the RootKey with a key encryption key (KEK). This is a bit of
// security theatre for local on-disk key material, but gives us a shim for
Expand Down
63 changes: 63 additions & 0 deletions nomad/keyring_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -360,3 +360,66 @@ func (k *Keyring) Delete(args *structs.KeyringDeleteRootKeyRequest, reply *struc
reply.Index = index
return nil
}

// ListPublic signing keys used for workload identities. This RPC is used to
// back a JWKS endpoint.
//
// Unauthenticated because public keys are not sensitive.
func (k *Keyring) ListPublic(args *structs.GenericRequest, reply *structs.KeyringListPublicResponse) error {

// JWKS is a public endpoint: intentionally ignore auth errors and only
// authenticate to measure rate metrics.
k.srv.Authenticate(k.ctx, args)
if done, err := k.srv.forward("Keyring.ListPublic", args, args, reply); done {
return err
}
k.srv.MeasureRPCRate("keyring", structs.RateMetricList, args)

defer metrics.MeasureSince([]string{"nomad", "keyring", "list_public"}, time.Now())

// Expose root_key_rotation_threshold so consumers can determine reasonable
// cache settings.
reply.RotationThreshold = k.srv.config.RootKeyRotationThreshold

// Setup the blocking query
opts := blockingOptions{
queryOpts: &args.QueryOptions,
queryMeta: &reply.QueryMeta,
run: func(ws memdb.WatchSet, s *state.StateStore) error {

// retrieve all the key metadata
snap, err := k.srv.fsm.State().Snapshot()
if err != nil {
return err
}
iter, err := snap.RootKeyMetas(ws)
if err != nil {
return err
}

pubKeys := []*structs.KeyringPublicKey{}
for {
raw := iter.Next()
if raw == nil {
break
}

keyMeta := raw.(*structs.RootKeyMeta)
if keyMeta.State == structs.RootKeyStateDeprecated {
// Only include valid keys
continue
}

pubKey, err := k.encrypter.GetPublicKey(keyMeta.KeyID)
if err != nil {
return err
}

pubKeys = append(pubKeys, pubKey)
}
reply.PublicKeys = pubKeys
return k.srv.replySetIndex(state.TableRootKeyMeta, &reply.QueryMeta)
},
}
return k.srv.blockingRPC(&opts)
}
57 changes: 57 additions & 0 deletions nomad/keyring_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
"github.com/shoenig/test/must"
"github.com/stretchr/testify/require"

"github.com/hashicorp/nomad/ci"
Expand Down Expand Up @@ -295,3 +296,59 @@ func TestKeyringEndpoint_Rotate(t *testing.T) {
gotKey := getResp.Key
require.Len(t, gotKey.Key, 32)
}

// TestKeyringEndpoint_ListPublic asserts the Keyring.ListPublic RPC returns
// all keys which may be in use for active crytpographic material (variables,
// valid JWTs).
func TestKeyringEndpoint_ListPublic(t *testing.T) {

ci.Parallel(t)
srv, rootToken, shutdown := TestACLServer(t, func(c *Config) {
c.NumSchedulers = 0 // Prevent automatic dequeue
})
defer shutdown()
testutil.WaitForLeader(t, srv.RPC)
codec := rpcClient(t, srv)

// Assert 1 key exists and normal fields are set
req := structs.GenericRequest{
QueryOptions: structs.QueryOptions{
Region: "global",
AuthToken: "ignored!",
},
}
var resp structs.KeyringListPublicResponse
must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.ListPublic", &req, &resp))
must.Eq(t, srv.config.RootKeyRotationThreshold, resp.RotationThreshold)
must.Len(t, 1, resp.PublicKeys)
must.NonZero(t, resp.Index)

// Rotate keys and assert there are now 2 keys
rotateReq := &structs.KeyringRotateRootKeyRequest{
WriteRequest: structs.WriteRequest{
Region: "global",
AuthToken: rootToken.SecretID,
},
}
var rotateResp structs.KeyringRotateRootKeyResponse
must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.Rotate", rotateReq, &rotateResp))
must.NotEq(t, resp.Index, rotateResp.Index)

// Verify we have a new key and the old one is inactive
var resp2 structs.KeyringListPublicResponse
must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.ListPublic", &req, &resp2))
must.Eq(t, srv.config.RootKeyRotationThreshold, resp2.RotationThreshold)
must.Len(t, 2, resp2.PublicKeys)
must.NonZero(t, resp2.Index)

found := false
for _, pk := range resp2.PublicKeys {
if pk.KeyID == resp.PublicKeys[0].KeyID {
must.False(t, found, must.Sprint("found the original public key twice"))
found = true
must.Eq(t, resp.PublicKeys[0], pk)
break
}
}
must.True(t, found, must.Sprint("original public key missing after rotation"))
}
Loading

0 comments on commit d14362e

Please sign in to comment.