Improved GetStateValidators (flashbots#368)

.gitignore

@@ -24,4 +24,4 @@
 .DS_Store
 /website-index.html
 /README.internal.md
-/internal/investigations/_*
+/internal/_investigations/

beaconclient/beacon_client_test.go

@@ -68,7 +68,7 @@ func TestBeaconInstance(t *testing.T) {
 		require.NoError(t, err)
 	})
 
-	vals, err := bc.FetchValidators(1)
+	vals, err := bc.GetStateValidators("1")
 	require.NoError(t, err)
 	require.Equal(t, 1, len(vals))
 	require.Contains(t, vals, types.PubkeyHex("0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"))
@@ -159,7 +159,7 @@ func TestFetchValidators(t *testing.T) {
 		backend := newTestBackend(t, 2)
 		backend.beaconInstances[0].MockFetchValidatorsErr = errTest
 		backend.beaconInstances[1].MockFetchValidatorsErr = errTest
-		status, err := backend.beaconClient.FetchValidators(1)
+		status, err := backend.beaconClient.GetStateValidators("1")
 		require.Error(t, err)
 		require.Nil(t, status)
 	})
@@ -179,7 +179,7 @@ func TestFetchValidators(t *testing.T) {
 		backend.beaconInstances[1].AddValidator(entry)
 		backend.beaconInstances[2].MockFetchValidatorsErr = errTest
 
-		validators, err := backend.beaconClient.FetchValidators(1)
+		validators, err := backend.beaconClient.GetStateValidators("1")
 		require.NoError(t, err)
 		require.Equal(t, 1, len(validators))
 		require.Contains(t, validators, types.PubkeyHex(testPubKey))
@@ -190,7 +190,7 @@ func TestFetchValidators(t *testing.T) {
 		backend.beaconInstances[2].AddValidator(entry)
 
 		t.Log(backend.beaconInstances[1].NumValidators())
-		validators, err = backend.beaconClient.FetchValidators(1)
+		validators, err = backend.beaconClient.GetStateValidators("1")
 		require.NoError(t, err)
 		require.Equal(t, 0, len(validators))
 	})

beaconclient/mock_beacon_instance.go

@@ -67,7 +67,7 @@ func (c *MockBeaconInstance) NumValidators() uint64 {
 	return uint64(len(c.validatorSet))
 }
 
-func (c *MockBeaconInstance) FetchValidators(headSlot uint64) (map[types.PubkeyHex]ValidatorResponseEntry, error) {
+func (c *MockBeaconInstance) GetStateValidators(stateID string) (map[types.PubkeyHex]ValidatorResponseEntry, error) {
 	c.addDelay()
 	return c.validatorSet, c.MockFetchValidatorsErr
 }

beaconclient/multi_beacon_client.go

@@ -27,8 +27,8 @@ type IMultiBeaconClient interface {
 	// SubscribeToPayloadAttributesEvents subscribes to payload attributes events to validate fields such as prevrandao and withdrawals
 	SubscribeToPayloadAttributesEvents(payloadAttrC chan PayloadAttributesEvent)
 
-	// FetchValidators returns all active and pending validators from the beacon node
-	FetchValidators(headSlot uint64) (map[types.PubkeyHex]ValidatorResponseEntry, error)
+	// GetStateValidators returns all active and pending validators from the beacon node
+	GetStateValidators(stateID string) (map[types.PubkeyHex]ValidatorResponseEntry, error)
 	GetProposerDuties(epoch uint64) (*ProposerDutiesResponse, error)
 	PublishBlock(block *common.SignedBeaconBlock) (code int, err error)
 	GetGenesis() (*GetGenesisResponse, error)
@@ -45,7 +45,7 @@ type IBeaconInstance interface {
 	CurrentSlot() (uint64, error)
 	SubscribeToHeadEvents(slotC chan HeadEventData)
 	SubscribeToPayloadAttributesEvents(slotC chan PayloadAttributesEvent)
-	FetchValidators(headSlot uint64) (map[types.PubkeyHex]ValidatorResponseEntry, error)
+	GetStateValidators(stateID string) (map[types.PubkeyHex]ValidatorResponseEntry, error)
 	GetProposerDuties(epoch uint64) (*ProposerDutiesResponse, error)
 	GetURI() string
 	PublishBlock(block *common.SignedBeaconBlock) (code int, err error)
@@ -149,15 +149,15 @@ func (c *MultiBeaconClient) SubscribeToPayloadAttributesEvents(slotC chan Payloa
 	}
 }
 
-func (c *MultiBeaconClient) FetchValidators(headSlot uint64) (map[types.PubkeyHex]ValidatorResponseEntry, error) {
+func (c *MultiBeaconClient) GetStateValidators(stateID string) (map[types.PubkeyHex]ValidatorResponseEntry, error) {
 	// return the first successful beacon node response
 	clients := c.beaconInstancesByLastResponse()
 
 	for i, client := range clients {
 		log := c.log.WithField("uri", client.GetURI())
 		log.Debug("fetching validators")
 
-		validators, err := client.FetchValidators(headSlot)
+		validators, err := client.GetStateValidators(stateID)
 		if err != nil {
 			log.WithError(err).Error("failed to fetch validators")
 			continue

beaconclient/prod_beacon_instance.go

@@ -109,18 +109,10 @@ func (c *ProdBeaconInstance) SubscribeToPayloadAttributesEvents(payloadAttribute
 	}
 }
 
-func (c *ProdBeaconInstance) FetchValidators(headSlot uint64) (map[types.PubkeyHex]ValidatorResponseEntry, error) {
-	vd, err := fetchAllValidators(c.beaconURI, headSlot)
-	if err != nil {
-		return nil, err
-	}
-
-	newValidatorSet := make(map[types.PubkeyHex]ValidatorResponseEntry)
-	for _, vs := range vd.Data {
-		newValidatorSet[types.NewPubkeyHex(vs.Validator.Pubkey)] = vs
-	}
-
-	return newValidatorSet, nil
+type GetStateValidatorsResponse struct {
+	ExecutionOptimistic bool `json:"execution_optimistic"`
+	Finalized           bool `json:"finalized"`
+	Data                []ValidatorResponseEntry
 }
 
 type ValidatorResponseEntry struct {
@@ -131,19 +123,32 @@ type ValidatorResponseEntry struct {
 }
 
 type ValidatorResponseValidatorData struct {
-	Pubkey string `json:"pubkey"`
-}
+	Pubkey                string `json:"pubkey"`
+	WithdrawalCredentials string `json:"withdrawal_credentials"`
+	EffectiveBalance      string `json:"effective_balance"`
+	Slashed               bool   `json:"slashed"`
+	ActivationEligibility uint64 `json:"activation_eligibility_epoch,string"`
+	ActivationEpoch       uint64 `json:"activation_epoch,string"`
+	ExitEpoch             uint64 `json:"exit_epoch,string"`
+	WithdrawableEpoch     uint64 `json:"withdrawable_epoch,string"`
+}
+
+// GetStateValidators loads all active and pending validators
+// https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
+func (c *ProdBeaconInstance) GetStateValidators(stateID string) (map[types.PubkeyHex]ValidatorResponseEntry, error) {
+	uri := fmt.Sprintf("%s/eth/v1/beacon/states/%s/validators?status=active,pending", c.beaconURI, stateID)
+	vd := new(GetStateValidatorsResponse)
+	_, err := fetchBeacon(http.MethodGet, uri, nil, vd)
+	if err != nil {
+		return nil, err
+	}
 
-type AllValidatorsResponse struct {
-	Data []ValidatorResponseEntry
-}
+	newValidatorSet := make(map[types.PubkeyHex]ValidatorResponseEntry)
+	for _, vs := range vd.Data {
+		newValidatorSet[types.NewPubkeyHex(vs.Validator.Pubkey)] = vs
+	}
 
-func fetchAllValidators(endpoint string, headSlot uint64) (*AllValidatorsResponse, error) {
-	uri := fmt.Sprintf("%s/eth/v1/beacon/states/%d/validators?status=active,pending", endpoint, headSlot)
-	// https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
-	vd := new(AllValidatorsResponse)
-	_, err := fetchBeacon(http.MethodGet, uri, nil, vd)
-	return vd, err
+	return newValidatorSet, nil
 }
 
 // SyncStatusPayload is the response payload for /eth/v1/node/syncing

beaconclient/util.go

@@ -9,7 +9,14 @@ import (
 	"net/http"
 )
 
-var ErrHTTPErrorResponse = errors.New("got an HTTP error response")
+var (
+	ErrHTTPErrorResponse = errors.New("got an HTTP error response")
+
+	StateIDHead      = "head"
+	StateIDGenesis   = "genesis"
+	StateIDFinalized = "finalized"
+	StateIDJustified = "justified"
+)
 
 func fetchBeacon(method, url string, payload, dst any) (code int, err error) {
 	var req *http.Request

services/housekeeper/housekeeper.go

@@ -173,7 +173,7 @@ func (hk *Housekeeper) updateKnownValidators() {
 	// Query beacon node for known validators
 	hk.log.Debug("Querying validators from beacon node... (this may take a while)")
 	timeStartFetching := time.Now()
-	validators, err := hk.beaconClient.FetchValidators(hk.headSlot.Load() - 1) // -1 to avoid "Invalid state ID: requested slot number is higher than head slot number" with multiple BNs
+	validators, err := hk.beaconClient.GetStateValidators(beaconclient.StateIDHead) // head is fastest
 	if err != nil {
 		hk.log.WithError(err).Error("failed to fetch validators from all beacon nodes")
 		return