Merge remote-tracking branch 'origin/main' into implement_eth_get_proof

README.md

@@ -134,7 +134,7 @@ datadir
         domain    # Latest State
         history   # Historical values 
         idx       # InvertedIndices: can search/filtering/union/intersect them - to find historical data. like eth_getLogs or trace_transaction
-        accessors # Additional (generated) indices of history - have "random-touch" read-pattern. They can serve only `Get` requests (no search/filters).
+        accessor # Additional (generated) indices of history - have "random-touch" read-pattern. They can serve only `Get` requests (no search/filters).
     txpool        # pending transactions. safe to remove.
     nodes         # p2p peers. safe to remove.
     temp          # used to sort data bigger than RAM. can grow to ~100gb. cleaned at startup.
@@ -155,11 +155,11 @@ datadir
         domain    # link to fast disk
         history   
         idx       
-        accessors 
+        accessor 
     temp # buffers to sort data >> RAM. sequential-buffered IO - is slow-disk-friendly   
 
 # Example: how to speedup history access: 
-#   - go step-by-step - first try store `accessors` on fast disk
+#   - go step-by-step - first try store `accessor` on fast disk
 #   - if speed is not good enough: `idx`
 #   - if still not enough: `history` 
 ```

accounts/abi/event.go

@@ -32,7 +32,7 @@ import (
 // don't get the signature canonical representation as the first LOG topic.
 type Event struct {
 	// Name is the event name used for internal representation. It's derived from
-	// the raw name and a suffix will be added in the case of a event overload.
+	// the raw name and a suffix will be added in the case of an event overload.
 	//
 	// e.g.
 	// These are two events that have the same name:

accounts/abi/reflect.go

@@ -27,7 +27,7 @@ import (
 	"strings"
 )
 
-// ConvertType converts an interface of a runtime type into a interface of the
+// ConvertType converts an interface of a runtime type into an interface of the
 // given type
 // e.g. turn
 // var fields []reflect.StructField

accounts/abi/type_test.go

@@ -28,7 +28,7 @@ import (
 	libcommon "github.com/erigontech/erigon-lib/common"
 )
 
-// typeWithoutStringer is a alias for the Type type which simply doesn't implement
+// typeWithoutStringer is an alias for the Type type which simply doesn't implement
 // the stringer interface to allow printing type details in the tests below.
 type typeWithoutStringer Type
 

cl/beacon/beaconhttp/api.go

@@ -99,11 +99,12 @@ func HandleEndpointFunc[T any](h EndpointHandlerFunc[T]) http.HandlerFunc {
 }
 
 func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		ans, err := h.Handle(w, r)
 		if err != nil {
 			var endpointError *EndpointError
-			if e, ok := err.(*EndpointError); ok {
+			var e *EndpointError
+			if errors.As(err, &e) {
 				endpointError = e
 			} else {
 				endpointError = WrapEndpointError(err)
@@ -131,14 +132,14 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc {
 				w.WriteHeader(200)
 			}
 		case strings.Contains(contentType, "application/octet-stream"):
-			sszMarshaler, ok := any(ans).(ssz.Marshaler)
+			sizeMarshaller, ok := any(ans).(ssz.Marshaler)
 			if !ok {
 				NewEndpointError(http.StatusBadRequest, ErrorSszNotSupported).WriteTo(w)
 				return
 			}
 			w.Header().Set("Content-Type", "application/octet-stream")
 			// TODO: we should probably figure out some way to stream this in the future :)
-			encoded, err := sszMarshaler.EncodeSSZ(nil)
+			encoded, err := sizeMarshaller.EncodeSSZ(nil)
 			if err != nil {
 				WrapEndpointError(err).WriteTo(w)
 				return
@@ -149,7 +150,7 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc {
 		default:
 			http.Error(w, "content type must include application/json, application/octet-stream, or text/event-stream, got "+contentType, http.StatusBadRequest)
 		}
-	})
+	}
 }
 
 func isNil[T any](t T) bool {

cl/beacon/handler/block_production.go

@@ -300,11 +300,16 @@ func (a *ApiHandler) GetEthV3ValidatorBlock(
 		)
 	}
 
-	// make a simple copy to the current head state
-	baseState, err := a.forkchoiceStore.GetStateAtBlockRoot(
-		baseBlockRoot,
-		true,
-	) // we start the block production from this state
+	var baseState *state.CachingBeaconState
+	if err := a.syncedData.ViewHeadState(func(headState *state.CachingBeaconState) error {
+		baseState, err = headState.Copy()
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
 
 	if err != nil {
 		return nil, err
@@ -1177,7 +1182,7 @@ func (a *ApiHandler) storeBlockAndBlobs(
 		return err
 	}
 
-	if err := a.forkchoiceStore.OnBlock(ctx, block, true, false, false); err != nil {
+	if err := a.forkchoiceStore.OnBlock(ctx, block, true, true, false); err != nil {
 		return err
 	}
 	finalizedBlockRoot := a.forkchoiceStore.FinalizedCheckpoint().Root

cl/beacon/handler/handler.go

@@ -228,12 +228,12 @@ func (a *ApiHandler) init() {
 			if a.routerCfg.Node {
 				r.Route("/node", func(r chi.Router) {
 					r.Get("/health", a.GetEthV1NodeHealth)
-					r.Get("/version", a.GetEthV1NodeVersion)
-					r.Get("/peer_count", a.GetEthV1NodePeerCount)
-					r.Get("/peers", a.GetEthV1NodePeersInfos)
-					r.Get("/peers/{peer_id}", a.GetEthV1NodePeerInfos)
-					r.Get("/identity", a.GetEthV1NodeIdentity)
-					r.Get("/syncing", a.GetEthV1NodeSyncing)
+					r.Get("/version", beaconhttp.HandleEndpointFunc(a.GetEthV1NodeVersion))
+					r.Get("/peer_count", beaconhttp.HandleEndpointFunc(a.GetEthV1NodePeerCount))
+					r.Get("/peers", beaconhttp.HandleEndpointFunc(a.GetEthV1NodePeersInfos))
+					r.Get("/peers/{peer_id}", beaconhttp.HandleEndpointFunc(a.GetEthV1NodePeerInfos))
+					r.Get("/identity", beaconhttp.HandleEndpointFunc(a.GetEthV1NodeIdentity))
+					r.Get("/syncing", beaconhttp.HandleEndpointFunc(a.GetEthV1NodeSyncing))
 				})
 			}
 

cl/beacon/handler/node.go

@@ -17,7 +17,7 @@
 package handler
 
 import (
-	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"runtime"
@@ -60,37 +60,29 @@ func (a *ApiHandler) GetEthV1NodeHealth(w http.ResponseWriter, r *http.Request)
 	w.WriteHeader(http.StatusOK)
 }
 
-func (a *ApiHandler) GetEthV1NodeVersion(w http.ResponseWriter, r *http.Request) {
+func (a *ApiHandler) GetEthV1NodeVersion(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
 	// Get OS and Arch
-	if err := json.NewEncoder(w).Encode(map[string]interface{}{
-		"data": map[string]interface{}{
-			"version": fmt.Sprintf("Caplin/%s %s/%s", a.version, runtime.GOOS, runtime.GOARCH),
-		},
-	}); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
+	return newBeaconResponse(map[string]interface{}{
+		"version": fmt.Sprintf("Caplin/%s %s/%s", a.version, runtime.GOOS, runtime.GOARCH),
+	}), nil
 }
 
-func (a *ApiHandler) GetEthV1NodePeerCount(w http.ResponseWriter, r *http.Request) {
+func (a *ApiHandler) GetEthV1NodePeerCount(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
 	ret, err := a.sentinel.GetPeers(r.Context(), &sentinel.EmptyMessage{})
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+		return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err)
 	}
+
 	// all fields should be converted to string
-	if err := json.NewEncoder(w).Encode(map[string]interface{}{
-		"data": map[string]interface{}{
-			"connected":     strconv.FormatUint(ret.Connected, 10),
-			"disconnected":  strconv.FormatUint(ret.Disconnected, 10),
-			"connecting":    strconv.FormatUint(ret.Connecting, 10),
-			"disconnecting": strconv.FormatUint(ret.Disconnecting, 10),
-		},
-	}); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
+	return newBeaconResponse(map[string]interface{}{
+		"connected":     strconv.FormatUint(ret.Connected, 10),
+		"disconnected":  strconv.FormatUint(ret.Disconnected, 10),
+		"connecting":    strconv.FormatUint(ret.Connecting, 10),
+		"disconnecting": strconv.FormatUint(ret.Disconnecting, 10),
+	}), nil
 }
 
-func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Request) {
+func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
 	state := r.URL.Query().Get("state")
 	direction := r.URL.Query().Get("direction")
 
@@ -107,8 +99,7 @@ func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Reque
 		State:     stateIn,
 	})
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+		return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err)
 	}
 	peers := make([]peer, 0, len(ret.Peers))
 	for i := range ret.Peers {
@@ -121,80 +112,64 @@ func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Reque
 			AgentVersion:       ret.Peers[i].AgentVersion,
 		})
 	}
-	if err := json.NewEncoder(w).Encode(map[string]interface{}{
-		"data": peers,
-	}); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
+
+	return newBeaconResponse(peers), nil
 }
 
-func (a *ApiHandler) GetEthV1NodePeerInfos(w http.ResponseWriter, r *http.Request) {
+func (a *ApiHandler) GetEthV1NodePeerInfos(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
 	pid, err := beaconhttp.StringFromRequest(r, "peer_id")
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
+		return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err)
 	}
 	ret, err := a.sentinel.PeersInfo(r.Context(), &sentinel.PeersInfoRequest{})
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+		return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err)
 	}
 	// find the peer with matching enr
 	for _, p := range ret.Peers {
 		if p.Pid == pid {
-			if err := json.NewEncoder(w).Encode(map[string]interface{}{
-				"data": peer{
-					PeerID:             p.Pid,
-					State:              p.State,
-					Enr:                p.Enr,
-					LastSeenP2PAddress: p.Address,
-					Direction:          p.Direction,
-					AgentVersion:       p.AgentVersion,
-				},
-			}); err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-			}
-			return
+			return newBeaconResponse(peer{
+				PeerID:             p.Pid,
+				State:              p.State,
+				Enr:                p.Enr,
+				LastSeenP2PAddress: p.Address,
+				Direction:          p.Direction,
+				AgentVersion:       p.AgentVersion,
+			}), nil
 		}
 	}
-	http.Error(w, "peer not found", http.StatusNotFound)
+
+	return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("peer not found"))
 }
 
-func (a *ApiHandler) GetEthV1NodeIdentity(w http.ResponseWriter, r *http.Request) {
+func (a *ApiHandler) GetEthV1NodeIdentity(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
 	id, err := a.sentinel.Identity(r.Context(), &sentinel.EmptyMessage{})
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+		return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err)
 	}
-	if err := json.NewEncoder(w).Encode(map[string]interface{}{
-		"data": map[string]interface{}{
-			"peer_id":             id.Pid,
-			"enr":                 id.Enr,
-			"p2p_addresses":       id.P2PAddresses,
-			"discovery_addresses": id.DiscoveryAddresses,
-			"metadata": map[string]interface{}{
-				"seq":      strconv.FormatUint(id.Metadata.Seq, 10),
-				"attnets":  id.Metadata.Attnets,
-				"syncnets": id.Metadata.Syncnets,
-			},
+
+	return newBeaconResponse(map[string]interface{}{
+		"peer_id":             id.Pid,
+		"enr":                 id.Enr,
+		"p2p_addresses":       id.P2PAddresses,
+		"discovery_addresses": id.DiscoveryAddresses,
+		"metadata": map[string]interface{}{
+			"seq":      strconv.FormatUint(id.Metadata.Seq, 10),
+			"attnets":  id.Metadata.Attnets,
+			"syncnets": id.Metadata.Syncnets,
 		},
-	}); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
+	}), nil
 }
 
-func (a *ApiHandler) GetEthV1NodeSyncing(w http.ResponseWriter, r *http.Request) {
+func (a *ApiHandler) GetEthV1NodeSyncing(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
 	currentSlot := a.ethClock.GetCurrentSlot()
 
-	if err := json.NewEncoder(w).Encode(map[string]interface{}{
-		"data": map[string]interface{}{
+	return newBeaconResponse(
+		map[string]interface{}{
 			"head_slot":     strconv.FormatUint(a.syncedData.HeadSlot(), 10),
 			"sync_distance": strconv.FormatUint(currentSlot-a.syncedData.HeadSlot(), 10),
 			"is_syncing":    a.syncedData.Syncing(),
 			"is_optimistic": false, // needs to change
 			"el_offline":    false,
-		},
-	}); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
+		}), nil
 }

cl/clparams/config.go

@@ -41,12 +41,15 @@ import (
 var LatestStateFileName = "latest.ssz_snappy"
 
 type CaplinConfig struct {
-	Backfilling               bool
-	BlobBackfilling           bool
+	// Archive related config
+	ArchiveBlocks             bool
+	ArchiveBlobs              bool
+	ArchiveStates             bool
+	ImmediateBlobsBackfilling bool
 	BlobPruningDisabled       bool
-	Archive                   bool
 	SnapshotGenerationEnabled bool
-	NetworkId                 NetworkType
+	// Network related config
+	NetworkId NetworkType
 	// DisableCheckpointSync is optional and is used to disable checkpoint sync used by default in the node
 	DisabledCheckpointSync bool
 	// CaplinMeVRelayUrl is optional and is used to connect to the external builder service.
@@ -341,7 +344,11 @@ var ConfigurableCheckpointsURLs = []string{}
 // MinEpochsForBlockRequests  equal to MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT / 2
 func (b *BeaconChainConfig) MinEpochsForBlockRequests() uint64 {
 	return b.MinValidatorWithdrawabilityDelay + (b.ChurnLimitQuotient)/2
+}
 
+// MinSlotsForBlobRequests  equal to MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUEST * SLOTS_PER_EPOCH
+func (b *BeaconChainConfig) MinSlotsForBlobsSidecarsRequest() uint64 {
+	return b.MinEpochsForBlobsSidecarsRequest * b.SlotsPerEpoch
 }
 
 type ConfigByte byte

cl/cltypes/block_production.go

@@ -25,7 +25,7 @@ import (
 )
 
 // BlindOrExecutionBeaconBlock is a union type that can be either a BlindedBeaconBlock or a BeaconBlock, depending on the context.
-// It's a intermediate type used in the block production process.
+// It's an intermediate type used in the block production process.
 type BlindOrExecutionBeaconBlock struct {
 	Slot          uint64         `json:"-"`
 	ProposerIndex uint64         `json:"-"`

cl/cltypes/solid/hash_list.go

@@ -109,6 +109,12 @@ func (h *hashList) CopyTo(t IterableSSZ[libcommon.Hash]) {
 			tu.MerkleTree = &merkle_tree.MerkleTree{}
 		}
 		h.MerkleTree.CopyInto(tu.MerkleTree)
+		// make the leaf function on the new buffer
+		tu.MerkleTree.SetComputeLeafFn(func(idx int, out []byte) {
+			copy(out, tu.u[idx*length.Hash:])
+		})
+	} else {
+		tu.MerkleTree = nil
 	}
 	copy(tu.u, h.u)
 }

cl/cltypes/solid/uint64slice_byte.go

@@ -81,6 +81,11 @@ func (arr *byteBasedUint64Slice) CopyTo(target *byteBasedUint64Slice) {
 			target.MerkleTree = &merkle_tree.MerkleTree{}
 		}
 		arr.MerkleTree.CopyInto(target.MerkleTree)
+		target.SetComputeLeafFn(func(idx int, out []byte) {
+			copy(out, target.u[idx*length.Hash:])
+		})
+	} else {
+		target.MerkleTree = nil
 	}
 
 	target.u = target.u[:len(arr.u)]