From e97f61e04aa35153579a410243efa854e83d9724 Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Sun, 19 May 2024 13:43:59 +0200 Subject: [PATCH] geth: add flag to control auto db compaction --- cmd/geth/chaincmd.go | 6 +++--- cmd/geth/main.go | 1 + cmd/utils/flags.go | 21 ++++++++++++++++++--- core/rawdb/database.go | 11 +++++++---- eth/backend.go | 2 +- eth/ethconfig/config.go | 2 ++ ethdb/pebble/pebble.go | 7 +++++-- node/node.go | 8 ++++++-- node/node_test.go | 10 ++++------ 9 files changed, 47 insertions(+), 21 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d333c17559..209fdd0b1b 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -221,7 +221,7 @@ func initGenesis(ctx *cli.Context) error { overrides.OverrideVerkle = &v } for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) + chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false, false) if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -258,7 +258,7 @@ func dumpGenesis(ctx *cli.Context) error { // dump whatever already exists in the datadir stack, _ := makeConfigNode(ctx) for _, name := range []string{"chaindata", "lightchaindata"} { - db, err := stack.OpenDatabase(name, 0, 0, "", true) + db, err := stack.OpenDatabase(name, 0, 0, "", true, false) if err != nil { if !os.IsNotExist(err) { return err @@ -561,7 +561,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth default: return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) } - var conf = &state.DumpConfig{ + conf := &state.DumpConfig{ SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name), diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d987364ca1..7d04a4707d 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -117,6 +117,7 @@ var ( utils.CacheLogSizeFlag, utils.FDLimitFlag, utils.CryptoKZGFlag, + utils.DBDisableAutoCompFlag, utils.ListenPortFlag, utils.DiscoveryPortFlag, utils.MaxPeersFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f4ecd35287..e60d016bdb 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -458,6 +458,12 @@ var ( Category: flags.PerfCategory, } + DBDisableAutoCompFlag = &cli.BoolFlag{ + Name: "db.disable-auto-comp", + Usage: "Disables database auto-compaction (pebble only)", + Category: flags.PerfCategory, + } + // Miner settings MiningEnabledFlag = &cli.BoolFlag{ Name: "mine", @@ -1756,6 +1762,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.DatabaseFreezer = ctx.String(AncientFlag.Name) } + if flag := DBDisableAutoCompFlag.Name; ctx.IsSet(flag) { + val := ctx.Bool(flag) + cfg.DisableAutomaticDBCompactions = val + if val { + log.Info("Disabling automatic database compaction") + } + // Don't log default behavior. + } + if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } @@ -1947,7 +1962,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { cfg.Genesis = nil // fallback to db content - //validate genesis has PoS enabled in block 0 + // validate genesis has PoS enabled in block 0 genesis, err := core.ReadGenesis(chaindb) if err != nil { Fatalf("Could not read genesis from database: %v", err) @@ -2140,9 +2155,9 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. } chainDb = remotedb.New(client) case ctx.String(SyncModeFlag.Name) == "light": - chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) + chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly, false) default: - chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly) + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly, false) } if err != nil { Fatalf("Could not open database: %v", err) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 27a9ec7412..1ff2875921 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -324,8 +324,8 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r // NewPebbleDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { - db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral) +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool, noAutoComp bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral, noAutoComp) if err != nil { return nil, err } @@ -366,6 +366,9 @@ type OpenOptions struct { // Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of // a crash is not important. This option should typically be used in tests. Ephemeral bool + + // Disable auto-compaction (PebbleDB only) + NoAutoCompaction bool } // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. @@ -387,7 +390,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { } if o.Type == dbPebble || existingDb == dbPebble { log.Info("Using pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral, o.NoAutoCompaction) } if o.Type == dbLeveldb || existingDb == dbLeveldb { log.Info("Using leveldb as the backing database") @@ -395,7 +398,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { } // No pre-existing database, no user-requested one either. Default to Pebble. log.Info("Defaulting to pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral, o.NoAutoCompaction) } // Open opens both a disk-based key-value database such as leveldb or pebble, but also diff --git a/eth/backend.go b/eth/backend.go index aedbd01634..21f6a66518 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -136,7 +136,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) + chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false, config.DisableAutomaticDBCompactions) if err != nil { return nil, err } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index fa1bfd4642..3d6c94cd0e 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -178,6 +178,8 @@ type Config struct { RollupDisableTxPoolGossip bool RollupDisableTxPoolAdmission bool RollupHaltOnIncompatibleProtocolVersion string + + DisableAutomaticDBCompactions bool } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index af4686cf5b..6aecc0e01e 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -135,7 +135,7 @@ func (l panicLogger) Fatalf(format string, args ...interface{}) { // New returns a wrapped pebble DB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool) (*Database, error) { +func New(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool, noAutoComp bool) (*Database, error) { // Ensure we have some minimal caching and file guarantees if cache < minCache { cache = minCache @@ -144,7 +144,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e handles = minHandles } logger := log.New("database", file) - logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) + logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles, + "no_auto_compaction", noAutoComp) // The max memtable size is limited by the uint32 offsets stored in // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry. @@ -219,6 +220,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e WriteStallEnd: db.onWriteStallEnd, }, Logger: panicLogger{}, // TODO(karalabe): Delete when this is upstreamed in Pebble + + DisableAutomaticCompactions: noAutoComp, } // Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130 // for more details. diff --git a/node/node.go b/node/node.go index 4dc856c345..f66f64575d 100644 --- a/node/node.go +++ b/node/node.go @@ -713,7 +713,7 @@ func (n *Node) EventMux() *event.TypeMux { // OpenDatabase opens an existing database with the given name (or creates one if no // previous can be found) from within the node's instance directory. If the node is // ephemeral, a memory database is returned. -func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool, noAutoComp bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -732,6 +732,8 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r Cache: cache, Handles: handles, ReadOnly: readonly, + + NoAutoCompaction: noAutoComp, }) } @@ -746,7 +748,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool, noAutoComp bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -765,6 +767,8 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient Cache: cache, Handles: handles, ReadOnly: readonly, + + NoAutoCompaction: noAutoComp, }) } diff --git a/node/node_test.go b/node/node_test.go index 04810a815b..7c3cde7a03 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -34,9 +34,7 @@ import ( "github.com/stretchr/testify/assert" ) -var ( - testNodeKey, _ = crypto.GenerateKey() -) +var testNodeKey, _ = crypto.GenerateKey() func testNodeConfig() *Config { return &Config{ @@ -152,7 +150,7 @@ func TestNodeCloseClosesDB(t *testing.T) { stack, _ := New(testNodeConfig()) defer stack.Close() - db, err := stack.OpenDatabase("mydb", 0, 0, "", false) + db, err := stack.OpenDatabase("mydb", 0, 0, "", false, true) if err != nil { t.Fatal("can't open DB:", err) } @@ -175,7 +173,7 @@ func TestNodeOpenDatabaseFromLifecycleStart(t *testing.T) { var err error stack.RegisterLifecycle(&InstrumentedService{ startHook: func() { - db, err = stack.OpenDatabase("mydb", 0, 0, "", false) + db, err = stack.OpenDatabase("mydb", 0, 0, "", false, true) if err != nil { t.Fatal("can't open DB:", err) } @@ -196,7 +194,7 @@ func TestNodeOpenDatabaseFromLifecycleStop(t *testing.T) { stack.RegisterLifecycle(&InstrumentedService{ stopHook: func() { - db, err := stack.OpenDatabase("mydb", 0, 0, "", false) + db, err := stack.OpenDatabase("mydb", 0, 0, "", false, true) if err != nil { t.Fatal("can't open DB:", err) }