Flag to enable compaction during initial sync

src/config.rs

@@ -41,6 +41,12 @@ pub struct Config {
     pub electrum_banner: String,
     pub electrum_rpc_logging: Option<RpcLogging>,
 
+    /// Enable compaction during initial sync
+    ///
+    /// By default compaction is off until initial sync is finished for performance reasons,
+    /// however, this requires much more disk space.
+    pub initial_sync_compaction: bool,
+
     #[cfg(feature = "liquid")]
     pub parent_network: BNetwork,
     #[cfg(feature = "liquid")]
@@ -191,6 +197,10 @@ impl Config {
                     .long("electrum-rpc-logging")
                     .help(&rpc_logging_help)
                     .takes_value(true),
+            ).arg(
+                Arg::with_name("initial_sync_compaction")
+                    .long("initial-sync-compaction")
+                    .help("Perform compaction during initial sync (slower but less disk space required)")
             );
 
         #[cfg(unix)]
@@ -403,6 +413,7 @@ impl Config {
             index_unspendables: m.is_present("index_unspendables"),
             cors: m.value_of("cors").map(|s| s.to_string()),
             precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()),
+            initial_sync_compaction: m.is_present("initial_sync_compaction"),
 
             #[cfg(feature = "liquid")]
             parent_network,

src/new_index/db.rs

@@ -90,7 +90,7 @@ impl DB {
         db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
         db_opts.set_target_file_size_base(1_073_741_824);
         db_opts.set_write_buffer_size(256 << 20);
-        db_opts.set_disable_auto_compactions(true); // for initial bulk load
+        db_opts.set_disable_auto_compactions(!config.initial_sync_compaction); // for initial bulk load
 
         // db_opts.set_advise_random_on_open(???);
         db_opts.set_compaction_readahead_size(1 << 20);
@@ -154,7 +154,7 @@ impl DB {
     }
 
     pub fn write(&self, mut rows: Vec<DBRow>, flush: DBFlush) {
-        debug!(
+        log::trace!(
             "writing {} rows to {:?}, flush={:?}",
             rows.len(),
             self.db,

src/new_index/fetch.rs

@@ -99,6 +99,7 @@ fn bitcoind_fetcher(
                 sender
                     .send(block_entries)
                     .expect("failed to send fetched blocks");
+                log::debug!("last fetch {:?}", entries.last());
             }
         }),
     ))

tests/common.rs

@@ -113,6 +113,7 @@ impl TestRunner {
             asset_db_path: None, // XXX
             #[cfg(feature = "liquid")]
             parent_network: bitcoin::Network::Regtest,
+            initial_sync_compaction: false,
             //#[cfg(feature = "electrum-discovery")]
             //electrum_public_hosts: Option<crate::electrum::ServerHosts>,
             //#[cfg(feature = "electrum-discovery")]