add CLI switch for adjusting Sideband upgrade batch sizes (#1684)

* resize batching for sideband upgrade to improve UI and RPC responsiveness

* Command line switch for configuring sideband batch_size, default is 512

* Simplification and updating max to use batch_size

* formatting :/

* cleanup

* Apply flags at node creation time

* Properly display options for unchecked_drop

nano/nano_node/daemon.cpp

@@ -133,10 +133,9 @@ void nano_daemon::daemon::run (boost::filesystem::path const & data_path, nano::
 		nano::node_init init;
 		try
 		{
-			auto node (std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, opencl_work, true));
+			auto node (std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, opencl_work, flags));
 			if (!init.error ())
 			{
-				node->flags = flags;
 				node->start ();
 				std::unique_ptr<nano::rpc> rpc = get_rpc (io_ctx, *node, config.rpc);
 				if (rpc)

nano/nano_node/entry.cpp

@@ -29,7 +29,9 @@ int main (int argc, char * const * argv)
 		("disable_wallet_bootstrap", "Disables wallet lazy bootstrap")
 		("disable_bootstrap_listener", "Disables bootstrap listener (incoming connections)")
 		("disable_unchecked_cleaning", "Disables periodic cleaning of old records from unchecked table")
+		("disable_unchecked_drop", "Disables drop of unchecked table at startup")
 		("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits")
+		("batch_size",boost::program_options::value<std::size_t> (), "Increase sideband batch size, default 512")
 		("debug_block_count", "Display the number of block")
 		("debug_bootstrap_generate", "Generate bootstrap sequence of blocks")
 		("debug_dump_representatives", "List representatives and weights")
@@ -80,19 +82,24 @@ int main (int argc, char * const * argv)
 
 	boost::filesystem::path data_path ((data_path_it != vm.end ()) ? data_path_it->second.as<std::string> () : nano::working_path ());
 	auto ec = nano::handle_node_options (vm);
-
 	if (ec == nano::error_cli::unknown_command)
 	{
 		if (vm.count ("daemon") > 0)
 		{
 			nano_daemon::daemon daemon;
 			nano::node_flags flags;
+			auto batch_size_it = vm.find ("batch_size");
+			if (batch_size_it != vm.end ())
+			{
+				flags.sideband_batch_size = batch_size_it->second.as<size_t> ();
+			}
 			flags.disable_backup = (vm.count ("disable_backup") > 0);
 			flags.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
 			flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
 			flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
 			flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
 			flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
+			flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
 			flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
 			daemon.run (data_path, flags);
 		}

nano/nano_wallet/entry.cpp

@@ -243,7 +243,7 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost
 		                                                       : std::function<boost::optional<uint64_t> (nano::uint256_union const &)> (nullptr));
 		nano::alarm alarm (io_ctx);
 		nano::node_init init;
-		node = std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, work, true);
+		node = std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, work, flags);
 		if (!init.error ())
 		{
 			auto wallet (node->wallets.open (config.wallet));
@@ -359,11 +359,18 @@ int main (int argc, char * const * argv)
 						data_path = nano::working_path ();
 					}
 					nano::node_flags flags;
+					auto batch_size_it = vm.find ("batch_size");
+					if (batch_size_it != vm.end ())
+					{
+						flags.sideband_batch_size = batch_size_it->second.as<size_t> ();
+					}
 					flags.disable_backup = (vm.count ("disable_backup") > 0);
 					flags.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
 					flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
 					flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
 					flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
+					flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
+					flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
 					flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
 					result = run_wallet (application, argc, argv, data_path, flags);
 				}

nano/node/lmdb.cpp

@@ -743,7 +743,7 @@ nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> nano::mdb_store
 	return nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> (nullptr);
 }
 
-nano::mdb_store::mdb_store (bool & error_a, nano::logging & logging_a, boost::filesystem::path const & path_a, int lmdb_max_dbs, bool drop_unchecked) :
+nano::mdb_store::mdb_store (bool & error_a, nano::logging & logging_a, boost::filesystem::path const & path_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size) :
 logging (logging_a),
 env (error_a, path_a, lmdb_max_dbs)
 {
@@ -783,9 +783,9 @@ env (error_a, path_a, lmdb_max_dbs)
 	}
 	if (slow_upgrade)
 	{
-		upgrades = std::thread ([this]() {
+		upgrades = std::thread ([this, batch_size]() {
 			nano::thread_role::set (nano::thread_role::name::slow_db_upgrade);
-			do_slow_upgrades ();
+			do_slow_upgrades (batch_size);
 		});
 	}
 }
@@ -1162,7 +1162,7 @@ void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_
 	mdb_drop (env.tx (transaction_a), unsynced, 1);
 }
 
-void nano::mdb_store::do_slow_upgrades ()
+void nano::mdb_store::do_slow_upgrades (size_t const batch_size)
 {
 	int version;
 	{
@@ -1185,7 +1185,7 @@ void nano::mdb_store::do_slow_upgrades ()
 		case 11:
 			break;
 		case 12:
-			upgrade_v12_to_v13 ();
+			upgrade_v12_to_v13 (batch_size);
 			break;
 		case 13:
 			break;
@@ -1205,10 +1205,9 @@ void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_
 	mdb_drop (env.tx (transaction_a), checksum, 1);
 }
 
-void nano::mdb_store::upgrade_v12_to_v13 ()
+void nano::mdb_store::upgrade_v12_to_v13 (size_t const batch_size)
 {
 	size_t cost (0);
-	size_t const max (16384);
 	nano::account account (0);
 	auto transaction (tx_begin_write ());
 	while (!stopped && account != nano::not_an_account)
@@ -1230,7 +1229,7 @@ void nano::mdb_store::upgrade_v12_to_v13 ()
 			nano::block_sideband sideband;
 			while (!stopped && !hash.is_zero ())
 			{
-				if (cost >= max)
+				if (cost >= batch_size)
 				{
 					BOOST_LOG (logging.log) << boost::str (boost::format ("Upgrading sideband information for account %1%... height %2%") % first.to_account ().substr (0, 24) % std::to_string (height));
 					auto tx (boost::polymorphic_downcast<nano::mdb_txn *> (transaction.impl.get ()));

nano/node/lmdb.hpp

@@ -150,7 +150,7 @@ class mdb_store : public block_store
 	friend class nano::block_predecessor_set;
 
 public:
-	mdb_store (bool &, nano::logging &, boost::filesystem::path const &, int lmdb_max_dbs = 128, bool drop_unchecked = false);
+	mdb_store (bool &, nano::logging &, boost::filesystem::path const &, int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512);
 	~mdb_store ();
 
 	nano::transaction tx_begin_write () override;
@@ -263,8 +263,8 @@ class mdb_store : public block_store
 	void upgrade_v9_to_v10 (nano::transaction const &);
 	void upgrade_v10_to_v11 (nano::transaction const &);
 	void upgrade_v11_to_v12 (nano::transaction const &);
-	void do_slow_upgrades ();
-	void upgrade_v12_to_v13 ();
+	void do_slow_upgrades (size_t const);
+	void upgrade_v12_to_v13 (size_t const);
 	bool full_sideband (nano::transaction const &);
 
 	// Requires a write transaction

nano/node/node.cpp

@@ -2055,12 +2055,13 @@ node (init_a, io_ctx_a, application_path_a, alarm_a, nano::node_config (peering_
 {
 }
 
-nano::node::node (nano::node_init & init_a, boost::asio::io_context & io_ctx_a, boost::filesystem::path const & application_path_a, nano::alarm & alarm_a, nano::node_config const & config_a, nano::work_pool & work_a, bool drop_unchecked) :
+nano::node::node (nano::node_init & init_a, boost::asio::io_context & io_ctx_a, boost::filesystem::path const & application_path_a, nano::alarm & alarm_a, nano::node_config const & config_a, nano::work_pool & work_a, nano::node_flags flags_a) :
 io_ctx (io_ctx_a),
 config (config_a),
 alarm (alarm_a),
 work (work_a),
-store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, config.logging, application_path_a / "data.ldb", config_a.lmdb_max_dbs, drop_unchecked)),
+flags (flags_a),
+store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, config.logging, application_path_a / "data.ldb", config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size)),
 store (*store_impl),
 wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (init_a.wallets_store_init, application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)),
 wallets_store (*wallets_store_impl),

nano/node/node.hpp

@@ -531,7 +531,7 @@ class node : public std::enable_shared_from_this<nano::node>
 {
 public:
 	node (nano::node_init &, boost::asio::io_context &, uint16_t, boost::filesystem::path const &, nano::alarm &, nano::logging const &, nano::work_pool &);
-	node (nano::node_init &, boost::asio::io_context &, boost::filesystem::path const &, nano::alarm &, nano::node_config const &, nano::work_pool &, bool drop_unchecked = false);
+	node (nano::node_init &, boost::asio::io_context &, boost::filesystem::path const &, nano::alarm &, nano::node_config const &, nano::work_pool &, nano::node_flags = nano::node_flags ());
 	~node ();
 	template <typename T>
 	void background (T action_a)

nano/node/nodeconfig.cpp

@@ -399,6 +399,8 @@ disable_legacy_bootstrap (false),
 disable_wallet_bootstrap (false),
 disable_bootstrap_listener (false),
 disable_unchecked_cleaning (false),
-fast_bootstrap (false)
+disable_unchecked_drop (true),
+fast_bootstrap (false),
+sideband_batch_size (512)
 {
 }

nano/node/nodeconfig.hpp

@@ -70,6 +70,8 @@ class node_flags
 	bool disable_wallet_bootstrap;
 	bool disable_bootstrap_listener;
 	bool disable_unchecked_cleaning;
+	bool disable_unchecked_drop;
 	bool fast_bootstrap;
+	size_t sideband_batch_size;
 };
 }