Skip to content

Commit

Permalink
Add cache validation for platform.GetValidatorsAt
Browse files Browse the repository at this point in the history
  • Loading branch information
StephenButtolph committed Jan 26, 2024
1 parent b81b936 commit 841a085
Show file tree
Hide file tree
Showing 8 changed files with 126 additions and 10 deletions.
4 changes: 4 additions & 0 deletions snow/validators/gvalidators/validator_state_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,7 @@ func (c *Client) GetValidatorSet(
}
return vdrs, nil
}

func (*Client) ValidateCachedGetValidatorSet(context.Context, uint64, ids.ID) error {
return nil
}
14 changes: 14 additions & 0 deletions snow/validators/mock_state.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions snow/validators/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ type State interface {
height uint64,
subnetID ids.ID,
) (map[ids.NodeID]*GetValidatorOutput, error)

ValidateCachedGetValidatorSet(
ctx context.Context,
targetHeight uint64,
subnetID ids.ID,
) error
}

type lockedState struct {
Expand Down Expand Up @@ -78,6 +84,10 @@ func (s *lockedState) GetValidatorSet(
return s.s.GetValidatorSet(ctx, height, subnetID)
}

func (*lockedState) ValidateCachedGetValidatorSet(context.Context, uint64, ids.ID) error {
return nil
}

type noValidators struct {
State
}
Expand Down
4 changes: 4 additions & 0 deletions snow/validators/test_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,7 @@ func (vm *TestState) GetValidatorSet(
}
return nil, errGetValidatorSet
}

func (*TestState) ValidateCachedGetValidatorSet(context.Context, uint64, ids.ID) error {
return nil
}
4 changes: 4 additions & 0 deletions snow/validators/traced_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,7 @@ func (s *tracedState) GetValidatorSet(

return s.s.GetValidatorSet(ctx, height, subnetID)
}

func (*tracedState) ValidateCachedGetValidatorSet(context.Context, uint64, ids.ID) error {
return nil
}
20 changes: 19 additions & 1 deletion vms/platformvm/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -2671,6 +2671,15 @@ func (v *GetValidatorsAtReply) MarshalJSON() ([]byte, error) {

m[vdr.NodeID] = vdrJSON
}
if v.ErrorString != "" {
return stdjson.Marshal(struct {
Validators map[ids.NodeID]*jsonGetValidatorOutput
ErrorString string
}{
Validators: m,
ErrorString: v.ErrorString,
})
}
return stdjson.Marshal(m)
}

Expand Down Expand Up @@ -2710,7 +2719,8 @@ func (v *GetValidatorsAtReply) UnmarshalJSON(b []byte) error {

// GetValidatorsAtReply is the response from GetValidatorsAt
type GetValidatorsAtReply struct {
Validators map[ids.NodeID]*validators.GetValidatorOutput
Validators map[ids.NodeID]*validators.GetValidatorOutput
ErrorString string
}

// GetValidatorsAt returns the weights of the validator set of a provided subnet
Expand All @@ -2733,6 +2743,14 @@ func (s *Service) GetValidatorsAt(r *http.Request, args *GetValidatorsAtArgs, re
if err != nil {
return fmt.Errorf("failed to get validator set: %w", err)
}
if err := s.vm.ValidateCachedGetValidatorSet(ctx, height, args.SubnetID); err != nil {
s.vm.ctx.Log.Error("invalid validator set",
zap.Stringer("subnetID", args.SubnetID),
zap.Uint64("height", height),
zap.Error(err),
)
reply.ErrorString = err.Error()
}
return nil
}

Expand Down
76 changes: 67 additions & 9 deletions vms/platformvm/validators/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ package validators

import (
"context"
"errors"
"fmt"
"reflect"
"time"

"github.com/ava-labs/avalanchego/cache"
Expand All @@ -30,13 +32,23 @@ const (
recentlyAcceptedWindowTTL = 2 * time.Minute
)

var _ validators.State = (*manager)(nil)
var (
_ validators.State = (*manager)(nil)

errInconsistentValidatorSet = errors.New("inconsistent validator set")
)

// Manager adds the ability to introduce newly accepted blocks IDs to the State
// interface.
type Manager interface {
validators.State

ValidateCachedGetValidatorSet(
ctx context.Context,
targetHeight uint64,
subnetID ids.ID,
) error

// OnAcceptedBlockID registers the ID of the latest accepted block.
// It is used to update the [recentlyAccepted] sliding window.
OnAcceptedBlockID(blkID ids.ID)
Expand Down Expand Up @@ -97,7 +109,7 @@ func NewManager(
state: state,
metrics: metrics,
clk: clk,
caches: make(map[ids.ID]cache.Cacher[uint64, map[ids.NodeID]*validators.GetValidatorOutput]),
caches: make(map[ids.ID]cache.Cacher[uint64, *cachedValidatorSet]),
recentlyAccepted: window.New[ids.ID](
window.Config{
Clock: clk,
Expand All @@ -121,12 +133,17 @@ type manager struct {
// Maps caches for each subnet that is currently tracked.
// Key: Subnet ID
// Value: cache mapping height -> validator set map
caches map[ids.ID]cache.Cacher[uint64, map[ids.NodeID]*validators.GetValidatorOutput]
caches map[ids.ID]cache.Cacher[uint64, *cachedValidatorSet]

// sliding window of blocks that were recently accepted
recentlyAccepted window.Window[ids.ID]
}

type cachedValidatorSet struct {
validatorSet map[ids.NodeID]*validators.GetValidatorOutput
calculatedHeight uint64
}

// GetMinimumHeight returns the height of the most recent block beyond the
// horizon of our recentlyAccepted window.
//
Expand Down Expand Up @@ -187,10 +204,9 @@ func (m *manager) GetValidatorSet(
subnetID ids.ID,
) (map[ids.NodeID]*validators.GetValidatorOutput, error) {
validatorSetsCache := m.getValidatorSetCache(subnetID)

if validatorSet, ok := validatorSetsCache.Get(targetHeight); ok {
m.metrics.IncValidatorSetsCached()
return validatorSet, nil
return validatorSet.validatorSet, nil
}

// get the start time to track metrics
Expand All @@ -211,7 +227,10 @@ func (m *manager) GetValidatorSet(
}

// cache the validator set
validatorSetsCache.Put(targetHeight, validatorSet)
validatorSetsCache.Put(targetHeight, &cachedValidatorSet{
validatorSet: validatorSet,
calculatedHeight: currentHeight,
})

duration := m.clk.Time().Sub(startTime)
m.metrics.IncValidatorSetsCreated()
Expand All @@ -220,18 +239,57 @@ func (m *manager) GetValidatorSet(
return validatorSet, nil
}

func (m *manager) getValidatorSetCache(subnetID ids.ID) cache.Cacher[uint64, map[ids.NodeID]*validators.GetValidatorOutput] {
func (m *manager) ValidateCachedGetValidatorSet(
ctx context.Context,
targetHeight uint64,
subnetID ids.ID,
) error {
validatorSetsCache := m.getValidatorSetCache(subnetID)
cachedValidatorSet, ok := validatorSetsCache.Get(targetHeight)
if !ok {
// If the validator set isn't cached, then there is nothing to check.
return nil
}

var (
validatorSet map[ids.NodeID]*validators.GetValidatorOutput
currentHeight uint64
err error
)
if subnetID == constants.PrimaryNetworkID {
validatorSet, currentHeight, err = m.makePrimaryNetworkValidatorSet(ctx, targetHeight)
} else {
validatorSet, currentHeight, err = m.makeSubnetValidatorSet(ctx, targetHeight, subnetID)
}
if err != nil {
return err
}

if reflect.DeepEqual(cachedValidatorSet.validatorSet, validatorSet) {
return nil
}

return fmt.Errorf("%w calculated for %s:%d at %d and %d",
errInconsistentValidatorSet,
subnetID,
targetHeight,
cachedValidatorSet.calculatedHeight,
currentHeight,
)
}

func (m *manager) getValidatorSetCache(subnetID ids.ID) cache.Cacher[uint64, *cachedValidatorSet] {
// Only cache tracked subnets
if subnetID != constants.PrimaryNetworkID && !m.cfg.TrackedSubnets.Contains(subnetID) {
return &cache.Empty[uint64, map[ids.NodeID]*validators.GetValidatorOutput]{}
return &cache.Empty[uint64, *cachedValidatorSet]{}
}

validatorSetsCache, exists := m.caches[subnetID]
if exists {
return validatorSetsCache
}

validatorSetsCache = &cache.LRU[uint64, map[ids.NodeID]*validators.GetValidatorOutput]{
validatorSetsCache = &cache.LRU[uint64, *cachedValidatorSet]{
Size: validatorSetsCacheSize,
}
m.caches[subnetID] = validatorSetsCache
Expand Down
4 changes: 4 additions & 0 deletions vms/platformvm/validators/test_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,8 @@ func (testManager) GetValidatorSet(context.Context, uint64, ids.ID) (map[ids.Nod
return nil, nil
}

func (testManager) ValidateCachedGetValidatorSet(context.Context, uint64, ids.ID) error {
return nil
}

func (testManager) OnAcceptedBlockID(ids.ID) {}

0 comments on commit 841a085

Please sign in to comment.