Merge branch 'erigontech:main' into curl

.github/workflows/backups-dashboards.yml

@@ -49,7 +49,7 @@ jobs:
     strategy:
       matrix:
         # For each dashboard add an entry in the list here below (and add an entry in the dictionary within the backup script too!)
-        dashboard: [erigon_custom_metrics]
+        dashboard: [erigon_custom_metrics,erigonQA]
     env:
       DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }}
       GH_TOKEN: ${{ secrets.GH_TOKEN }}

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ on:
       - reopened
       - synchronize
       - ready_for_review
+
   workflow_dispatch:
 
 concurrency:

.github/workflows/test-integration-caplin.yml

@@ -13,7 +13,7 @@ on:
       - reopened
       - synchronize
       - ready_for_review
-
+    
 jobs:
   tests:
     strategy:

.github/workflows/test-integration-erigon.yml

@@ -1,4 +1,5 @@
 name: Integration tests - ERIGON
+
 on:
   push:
     branches:
@@ -17,59 +18,114 @@ on:
   workflow_dispatch:
 
 jobs:
+  # 
+  # This first job is used to determine if changes are within out-of-scope dirs or files (in such a case integration tests are not run because they would be meaningless)
+  # NOTE: this logic is needed because the simple 'paths-ignore:' doesn't work since this workflow is set as a mandatory/required check for this repo
+  # - '**/.github/workflows/**' is currently commented to avoid unintended freeze in case of concurrent changes outside the excluded paths (further development will be done in due course)
+  # 
+  source-of-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changed_files: ${{ steps.filter.outputs.changed_files }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check for changes within out-of-scope dirs or files
+        id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            changed_files:
+              - 'dashboards/**'
+              # - '**/.github/workflows/**'
+              - '**/.github/workflows/backups-dashboards.yml'
+
   tests-mac-linux:
+    needs: source-of-changes
     strategy:
       matrix:
-        # list of os: https://github.com/actions/virtual-environments
         os:
           - ubuntu-22.04
           - macos-14
-          - ubuntu-latest-erigontests-large
+          # - ubuntu-latest-erigontests-large
     runs-on: ${{ matrix.os }}
 
     steps:
-      - name: declaring runners
+      - name: Declare runners
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: |
           set +x
           echo "I am being served by this runner: $RUNNER_NAME"
-      - uses: actions/checkout@v4
-      - run: git submodule update --init --recursive --force
-      - uses: actions/setup-go@v5
+          
+      - name: Checkout code
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/checkout@v4
+
+      - name: Update submodules
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        run: git submodule update --init --recursive --force
+
+      - name: Setup Go environment
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/setup-go@v5
         with:
           go-version: '1.23'
           cache: ${{ contains(fromJSON('["refs/heads/main","refs/heads/release/2.60","refs/heads/release/2.61"]'), github.ref) }}
+
       - name: Install dependencies on Linux
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && needs.source-of-changes.outputs.changed_files != 'true'
         run: sudo apt update && sudo apt install build-essential
-      - name: test-integration
+
+      - name: Run integration tests on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: GOGC=80 make test-integration
 
+      - name: This ${{ matrix.os }} check does not make sense for changes within out-of-scope directories
+        if: needs.source-of-changes.outputs.changed_files == 'true'
+        run: echo "This check does not make sense for changes within out-of-scope directories"
+
+
   tests-windows:
+    needs: source-of-changes
     strategy:
       matrix:
         os: [ windows-2022 ]
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
-      - run: git submodule update --init --recursive --force
-      - uses: actions/setup-go@v5
+      - name: Checkout code on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/checkout@v4
+
+      - name: Update submodules on Windows
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        run: git submodule update --init --recursive --force
+
+      - name: Setup Go environment on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/setup-go@v5
         with:
           go-version: '1.23'
 
-      - uses: actions/cache@v4
+      - name: Cache Chocolatey packages
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/cache@v4
         with:
           path: |
             C:\ProgramData\chocolatey\lib\mingw
             C:\ProgramData\chocolatey\lib\cmake
           key: chocolatey-${{ matrix.os }}
-      - name: Install dependencies
+
+      - name: Install dependencies on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: |
           choco upgrade mingw -y --no-progress --version 13.2.0
 
-      - name: test-integration
+      - name: Run integration tests on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: .\wmake.ps1 test-integration
 
-
-
-
+      - name: This ${{ matrix.os }} check does not make sense for changes within out-of-scope directories
+        if: needs.source-of-changes.outputs.changed_files == 'true'
+        run: echo "This check does not make sense for changes within out-of-scope directories"

README.md

@@ -1,8 +1,8 @@
 # Erigon
 
 Documentation: **[docs.erigon.tech](https://docs.erigon.tech)**
-Blog: **[erigon.substack.com](https://erigon.substack.com/)**
-Twitter: [x.com/ErigonEth](https://x.com/ErigonEth)
+Blog: **[erigon.tech/news](https://erigon.tech/news/)**
+X/Twitter: **[x.com/ErigonEth](https://x.com/ErigonEth)**
 
 Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency
 frontier.
@@ -722,14 +722,6 @@ Getting in touch
 The main discussions are happening on our Discord server. To get an invite, send an email to `bloxster [at] proton.me`
 with your name, occupation, a brief explanation of why you want to join the Discord, and how you heard about Erigon.
 
-### Blog
-
-**[erigon.substack.com](https://erigon.substack.com/)**
-
-### Twitter
-
-[x.com/ErigonEth](https://x.com/ErigonEth)
-
 ### Reporting security issues/concerns
 
 Send an email to `security [at] torquem.ch`.

cl/monitor/metrics.go

@@ -28,6 +28,8 @@ var (
 	batchVerificationThroughput    = metrics.GetOrCreateGauge("aggregation_per_signature")
 	blobVerificationTime           = metrics.GetOrCreateGauge("blob_verification_time")
 	executionTime                  = metrics.GetOrCreateGauge("execution_time")
+	executionClientInsertingBlocks = metrics.GetOrCreateGauge("execution_client_insert_blocks_time")
+	executionClientValidateChain   = metrics.GetOrCreateGauge("execution_client_validate_chain_time")
 
 	// Epoch processing metrics
 	EpochProcessingTime                     = metrics.GetOrCreateGauge("epoch_processing_time")
@@ -223,6 +225,14 @@ func ObserveBlobVerificationTime(startTime time.Time) {
 	blobVerificationTime.Set(microToMilli(time.Since(startTime).Microseconds()))
 }
 
-func ObserveExecutionTime(startTime time.Time) {
+func ObserveNewPayloadTime(startTime time.Time) {
 	executionTime.Set(microToMilli(time.Since(startTime).Microseconds()))
 }
+
+func ObserveExecutionClientInsertingBlocks(startTime time.Time) {
+	executionClientInsertingBlocks.Set(microToMilli(time.Since(startTime).Microseconds()))
+}
+
+func ObserveExecutionClientValidateChain(startTime time.Time) {
+	executionClientValidateChain.Set(microToMilli(time.Since(startTime).Microseconds()))
+}

cl/phase1/execution_client/execution_client_direct.go

@@ -22,11 +22,13 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"time"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/common/hexutility"
 	execution "github.com/erigontech/erigon-lib/gointerfaces/executionproto"
 	"github.com/erigontech/erigon/cl/cltypes"
+	"github.com/erigontech/erigon/cl/monitor"
 	"github.com/erigontech/erigon/core/types"
 	"github.com/erigontech/erigon/turbo/engineapi/engine_types"
 	"github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader"
@@ -66,20 +68,24 @@ func (cc *ExecutionClientDirect) NewPayload(
 		return PayloadStatusInvalidated, err
 	}
 
+	startInsertBlockAndWait := time.Now()
 	if err := cc.chainRW.InsertBlockAndWait(ctx, types.NewBlockFromStorage(payload.BlockHash, header, txs, nil, body.Withdrawals)); err != nil {
 		return PayloadStatusNone, err
 	}
+	monitor.ObserveExecutionClientInsertingBlocks(startInsertBlockAndWait)
 
 	headHeader := cc.chainRW.CurrentHeader(ctx)
 	if headHeader == nil || header.Number.Uint64() > headHeader.Number.Uint64()+1 {
 		// can't validate yet
 		return PayloadStatusNotValidated, nil
 	}
 
+	startValidateChain := time.Now()
 	status, _, _, err := cc.chainRW.ValidateChain(ctx, payload.BlockHash, payload.BlockNumber)
 	if err != nil {
 		return PayloadStatusNone, err
 	}
+	monitor.ObserveExecutionClientValidateChain(startValidateChain)
 	// check status
 	switch status {
 	case execution.ExecutionStatus_BadBlock, execution.ExecutionStatus_InvalidForkchoice:

cl/phase1/forkchoice/on_block.go

@@ -115,6 +115,7 @@ func (f *ForkChoiceStore) ProcessBlockExecution(ctx context.Context, block *clty
 
 	timeStartExec := time.Now()
 	payloadStatus, err := f.engine.NewPayload(ctx, block.Block.Body.ExecutionPayload, &block.Block.ParentRoot, versionedHashes, executionRequestsList)
+	monitor.ObserveNewPayloadTime(timeStartExec)
 	switch payloadStatus {
 	case execution_client.PayloadStatusInvalidated:
 		log.Warn("OnBlock: block is invalid", "block", libcommon.Hash(blockRoot), "err", err)
@@ -135,7 +136,6 @@ func (f *ForkChoiceStore) ProcessBlockExecution(ctx context.Context, block *clty
 	if err != nil {
 		return fmt.Errorf("newPayload failed: %v", err)
 	}
-	monitor.ObserveExecutionTime(timeStartExec)
 	return nil
 }
 
@@ -213,6 +213,7 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac
 		}
 		timeStartExec := time.Now()
 		payloadStatus, err := f.engine.NewPayload(ctx, block.Block.Body.ExecutionPayload, &block.Block.ParentRoot, versionedHashes, executionRequestsList)
+		monitor.ObserveNewPayloadTime(timeStartExec)
 		switch payloadStatus {
 		case execution_client.PayloadStatusNotValidated:
 			log.Debug("OnBlock: block is not validated yet", "block", libcommon.Hash(blockRoot))
@@ -239,7 +240,6 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac
 		if err != nil {
 			return fmt.Errorf("newPayload failed: %v", err)
 		}
-		monitor.ObserveExecutionTime(timeStartExec)
 	}
 	log.Trace("OnBlock: engine", "elapsed", time.Since(startEngine))
 	startStateProcess := time.Now()

cl/phase1/network/services/constants.go

@@ -29,8 +29,8 @@ const (
 	blobJobsIntervalTick          = 5 * time.Millisecond
 	singleAttestationIntervalTick = 10 * time.Millisecond
 	attestationJobsIntervalTick   = 100 * time.Millisecond
-	blockJobExpiry                = 4 * time.Second
-	blobJobExpiry                 = 4 * time.Second
+	blockJobExpiry                = 30 * time.Second
+	blobJobExpiry                 = 30 * time.Second
 	attestationJobExpiry          = 30 * time.Minute
 	singleAttestationJobExpiry    = 6 * time.Second
 )

cmd/prometheus/dashboards/erigon_internals.json

@@ -1582,7 +1582,7 @@
               }
             }
           ],
-          "title": "ValidateChain: time spent",
+          "title": "NewPayload: time spent",
           "type": "timeseries"
         },
         {

cmd/rpcdaemon/cli/config.go

@@ -123,7 +123,7 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().BoolVar(&cfg.GraphQLEnabled, "graphql", false, "enables graphql endpoint (disabled by default)")
 	rootCmd.PersistentFlags().Uint64Var(&cfg.Gascap, "rpc.gascap", 50_000_000, "Sets a cap on gas that can be used in eth_call/estimateGas")
 	rootCmd.PersistentFlags().Uint64Var(&cfg.MaxTraces, "trace.maxtraces", 200, "Sets a limit on traces that can be returned in trace_filter")
-	rootCmd.PersistentFlags().BoolVar(&polygonSync, "polygon.sync", false, "Enable if Erigon has been synced using the new polygon sync component")
+	rootCmd.PersistentFlags().BoolVar(&polygonSync, "polygon.sync", true, "Enable if Erigon has been synced using the new polygon sync component")
 
 	rootCmd.PersistentFlags().StringVar(&cfg.RpcAllowListFilePath, utils.RpcAccessListFlag.Name, "", "Specify granular (method-by-method) API allowlist")
 	rootCmd.PersistentFlags().UintVar(&cfg.RpcBatchConcurrency, utils.RpcBatchConcurrencyFlag.Name, 2, utils.RpcBatchConcurrencyFlag.Usage)

cmd/utils/flags.go

@@ -788,12 +788,14 @@ var (
 		Usage: "Ignore the bor block period and wait for 'blocksize' transactions (for testing purposes)",
 	}
 
+	// TODO - this is a depricated flag - should be removed
 	WithHeimdallMilestones = cli.BoolFlag{
 		Name:  "bor.milestone",
 		Usage: "Enabling bor milestone processing",
 		Value: true,
 	}
 
+	// TODO - this is a depricated flag - should be removed
 	WithHeimdallWaypoints = cli.BoolFlag{
 		Name:  "bor.waypoints",
 		Usage: "Enabling bor waypont recording",
@@ -803,8 +805,10 @@ var (
 	PolygonSyncFlag = cli.BoolFlag{
 		Name:  "polygon.sync",
 		Usage: "Enabling syncing using the new polygon sync component",
+		Value: true,
 	}
 
+	// TODO - this is a depricated flag - should be removed
 	PolygonSyncStageFlag = cli.BoolFlag{
 		Name:  "polygon.sync.stage",
 		Usage: "Enabling syncing with a stage that uses the polygon sync component",
@@ -1693,6 +1697,12 @@ func setBorConfig(ctx *cli.Context, cfg *ethconfig.Config, nodeConfig *nodecfg.C
 	cfg.WithHeimdallWaypointRecording = ctx.Bool(WithHeimdallWaypoints.Name)
 	cfg.PolygonSync = ctx.Bool(PolygonSyncFlag.Name)
 	cfg.PolygonSyncStage = ctx.Bool(PolygonSyncStageFlag.Name)
+
+	if cfg.PolygonSync {
+		cfg.WithHeimdallMilestones = false
+		cfg.WithHeimdallWaypointRecording = true
+	}
+
 	heimdall.RecordWayPoints(cfg.WithHeimdallWaypointRecording || cfg.PolygonSync || cfg.PolygonSyncStage)
 
 	chainConfig := params.ChainConfigByChainName(ctx.String(ChainFlag.Name))