From f1221210ba2467e1b1b6851ea8731df0031ebc8a Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Fri, 20 Sep 2024 10:08:52 +0200 Subject: [PATCH 1/2] Flag to enable compaction during initial sync While compaction during initial sync makes thing much slower, it may be preferred to not require to size the disk the double of the final required space. --- src/config.rs | 11 +++++++++++ src/new_index/db.rs | 2 +- tests/common.rs | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 8696ecf8f..380ba21dd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -41,6 +41,12 @@ pub struct Config { pub electrum_banner: String, pub electrum_rpc_logging: Option, + /// Enable compaction during initial sync + /// + /// By default compaction is off until initial sync is finished for performance reasons, + /// however, this requires much more disk space. + pub initial_sync_compaction: bool, + #[cfg(feature = "liquid")] pub parent_network: BNetwork, #[cfg(feature = "liquid")] @@ -191,6 +197,10 @@ impl Config { .long("electrum-rpc-logging") .help(&rpc_logging_help) .takes_value(true), + ).arg( + Arg::with_name("initial_sync_compaction") + .long("initial-sync-compaction") + .help("Perform compaction during initial sync (slower but less disk space required)") ); #[cfg(unix)] @@ -403,6 +413,7 @@ impl Config { index_unspendables: m.is_present("index_unspendables"), cors: m.value_of("cors").map(|s| s.to_string()), precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()), + initial_sync_compaction: m.is_present("initial_sync_compaction"), #[cfg(feature = "liquid")] parent_network, diff --git a/src/new_index/db.rs b/src/new_index/db.rs index f68da233c..77e1a66d4 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -90,7 +90,7 @@ impl DB { db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy); db_opts.set_target_file_size_base(1_073_741_824); db_opts.set_write_buffer_size(256 << 20); - db_opts.set_disable_auto_compactions(true); // for initial bulk load + db_opts.set_disable_auto_compactions(!config.initial_sync_compaction); // for initial bulk load // db_opts.set_advise_random_on_open(???); db_opts.set_compaction_readahead_size(1 << 20); diff --git a/tests/common.rs b/tests/common.rs index 23c7ce1e2..a3f38a412 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -113,6 +113,7 @@ impl TestRunner { asset_db_path: None, // XXX #[cfg(feature = "liquid")] parent_network: bitcoin::Network::Regtest, + initial_sync_compaction: false, //#[cfg(feature = "electrum-discovery")] //electrum_public_hosts: Option, //#[cfg(feature = "electrum-discovery")] From 6d182d8d0801a40c44a26000ae731a38c1d96218 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Fri, 20 Sep 2024 12:14:17 +0200 Subject: [PATCH 2/2] improve logging during initial sync it is less interesting to see how many rows are written in the db and more interesting knowing the last height indexed --- src/new_index/db.rs | 2 +- src/new_index/fetch.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 77e1a66d4..b6617d425 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -154,7 +154,7 @@ impl DB { } pub fn write(&self, mut rows: Vec, flush: DBFlush) { - debug!( + log::trace!( "writing {} rows to {:?}, flush={:?}", rows.len(), self.db, diff --git a/src/new_index/fetch.rs b/src/new_index/fetch.rs index 11843f5d7..d7637ee5c 100644 --- a/src/new_index/fetch.rs +++ b/src/new_index/fetch.rs @@ -99,6 +99,7 @@ fn bitcoind_fetcher( sender .send(block_entries) .expect("failed to send fetched blocks"); + log::debug!("last fetch {:?}", entries.last()); } }), ))