Lazy bootstrap retry limit (#2362)

- different retry limit for lazy destinations and possible links from state blocks (add retry limit to nano::bulk_pull)
- start lazy pulls for all known destinations in lazy bootstrap attempt
- dynamic bulk pull blocks limit for lazy bootstrap based on lazy blocks ratio (total processed blocks / lazy blocks)
- don't clean unchecked for longest bootstrap attempts
- use bootstrap network params
- improve lock in process_block_lazy ()
- increase max lazy bootstrap duration
- change max lazy keys limit
- increase bootstrap_max_new_connections
- fix: stop lazy bootstrap with stopped node
- fix: proper reuse of of connections after lazy pull stop

nano/node/bootstrap/bootstrap.hpp

@@ -46,7 +46,7 @@ class lazy_state_backlog_item final
 public:
 	nano::link link{ 0 };
 	nano::uint128_t balance{ 0 };
-	bool confirmed{ false };
+	unsigned retry_limit{ 0 };
 };
 class lazy_destinations_item final
 {
@@ -81,17 +81,17 @@ class bootstrap_attempt final : public std::enable_shared_from_this<bootstrap_at
 	void add_bulk_push_target (nano::block_hash const &, nano::block_hash const &);
 	void attempt_restart_check (nano::unique_lock<std::mutex> &);
 	void confirm_frontiers (nano::unique_lock<std::mutex> &);
-	bool process_block (std::shared_ptr<nano::block>, nano::account const &, uint64_t, bool, bool);
+	bool process_block (std::shared_ptr<nano::block>, nano::account const &, uint64_t, nano::bulk_pull::count_t, bool, unsigned);
 	/** Lazy bootstrap */
 	void lazy_run ();
 	void lazy_start (nano::hash_or_account const &, bool confirmed = true);
-	void lazy_add (nano::hash_or_account const &, bool = true);
+	void lazy_add (nano::hash_or_account const &, unsigned = std::numeric_limits<unsigned>::max ());
 	void lazy_requeue (nano::block_hash const &, nano::block_hash const &, bool);
 	bool lazy_finished ();
 	void lazy_pull_flush ();
 	void lazy_clear ();
-	bool process_block_lazy (std::shared_ptr<nano::block>, nano::account const &, uint64_t, bool);
-	void lazy_block_state (std::shared_ptr<nano::block>, bool);
+	bool process_block_lazy (std::shared_ptr<nano::block>, nano::account const &, uint64_t, nano::bulk_pull::count_t, unsigned);
+	void lazy_block_state (std::shared_ptr<nano::block>, unsigned);
 	void lazy_block_state_backlog_check (std::shared_ptr<nano::block>, nano::block_hash const &);
 	void lazy_backlog_cleanup ();
 	void lazy_destinations_increment (nano::account const &);
@@ -136,7 +136,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this<bootstrap_at
 	std::unordered_set<nano::block_hash> lazy_undefined_links;
 	std::unordered_map<nano::block_hash, nano::uint128_t> lazy_balances;
 	std::unordered_set<nano::block_hash> lazy_keys;
-	std::deque<std::pair<nano::hash_or_account, bool>> lazy_pulls;
+	std::deque<std::pair<nano::hash_or_account, unsigned>> lazy_pulls;
 	std::chrono::steady_clock::time_point last_lazy_flush{ std::chrono::steady_clock::now () };
 	class account_tag
 	{
@@ -150,7 +150,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this<bootstrap_at
 	boost::multi_index::ordered_non_unique<boost::multi_index::tag<count_tag>, boost::multi_index::member<lazy_destinations_item, uint64_t, &lazy_destinations_item::count>, std::greater<uint64_t>>,
 	boost::multi_index::hashed_unique<boost::multi_index::tag<account_tag>, boost::multi_index::member<lazy_destinations_item, nano::account, &lazy_destinations_item::account>>>>
 	lazy_destinations;
-	std::chrono::steady_clock::time_point last_lazy_destinations_flush{ std::chrono::steady_clock::time_point{} };
+	std::atomic<bool> lazy_destinations_flushed{ false };
 	std::mutex lazy_mutex;
 	// Wallet lazy bootstrap
 	std::deque<nano::account> wallet_accounts;
@@ -264,22 +264,22 @@ class bootstrap_limits final
 {
 public:
 	static constexpr double bootstrap_connection_scale_target_blocks = 50000.0;
+	static constexpr double bootstrap_connection_scale_target_blocks_lazy = bootstrap_connection_scale_target_blocks / 5;
 	static constexpr double bootstrap_connection_warmup_time_sec = 5.0;
 	static constexpr double bootstrap_minimum_blocks_per_sec = 10.0;
 	static constexpr double bootstrap_minimum_elapsed_seconds_blockrate = 0.02;
 	static constexpr double bootstrap_minimum_frontier_blocks_per_sec = 1000.0;
-	static constexpr unsigned bootstrap_frontier_retry_limit = 16;
-	static constexpr unsigned bootstrap_lazy_retry_limit = bootstrap_frontier_retry_limit * 10;
 	static constexpr double bootstrap_minimum_termination_time_sec = 30.0;
-	static constexpr unsigned bootstrap_max_new_connections = 10;
+	static constexpr unsigned bootstrap_max_new_connections = 32;
 	static constexpr size_t bootstrap_max_confirm_frontiers = 70;
 	static constexpr double required_frontier_confirmation_ratio = 0.8;
 	static constexpr unsigned frontier_confirmation_blocks_limit = 128 * 1024;
 	static constexpr unsigned requeued_pulls_limit = 256;
 	static constexpr unsigned requeued_pulls_limit_test = 2;
 	static constexpr unsigned bulk_push_cost_limit = 200;
 	static constexpr std::chrono::seconds lazy_flush_delay_sec = std::chrono::seconds (5);
-	static constexpr unsigned bootstrap_lazy_destinations_request_limit = 200;
-	static constexpr std::chrono::seconds lazy_destinations_flush_delay_sec = std::chrono::minutes (2);
+	static constexpr unsigned lazy_destinations_request_limit = 256 * 1024;
+	static constexpr uint64_t lazy_batch_pull_count_resize_blocks_limit = 4 * 1024 * 1024;
+	static constexpr double lazy_batch_pull_count_resize_ratio = 2.0;
 };
 }

nano/node/bootstrap/bootstrap_bulk_pull.cpp

@@ -3,13 +3,13 @@
 #include <nano/node/node.hpp>
 #include <nano/node/transport/tcp.hpp>
 
-nano::pull_info::pull_info (nano::hash_or_account const & account_or_head_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a, bool confirmed_head_a) :
+nano::pull_info::pull_info (nano::hash_or_account const & account_or_head_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a, unsigned retry_limit_a) :
 account_or_head (account_or_head_a),
 head (head_a),
 head_original (head_a),
 end (end_a),
 count (count_a),
-confirmed_head (confirmed_head_a)
+retry_limit (retry_limit_a)
 {
 }
 
@@ -53,7 +53,7 @@ nano::bulk_pull_client::~bulk_pull_client ()
 
 void nano::bulk_pull_client::request ()
 {
-	assert (!pull.head.is_zero () || !pull.confirmed_head);
+	assert (!pull.head.is_zero () || pull.retry_limit != std::numeric_limits<unsigned>::max ());
 	expected = pull.head;
 	nano::bulk_pull req;
 	if (pull.head == pull.head_original)
@@ -182,7 +182,7 @@ void nano::bulk_pull_client::received_type ()
 		case nano::block_type::not_a_block:
 		{
 			// Avoid re-using slow peers, or peers that sent the wrong blocks.
-			if (!connection->pending_stop && expected == pull.end)
+			if (!connection->pending_stop && (expected == pull.end || (pull.count != 0 && pull.count == pull_blocks)))
 			{
 				connection->attempt->pool_connection (connection);
 			}
@@ -216,7 +216,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e
 			}
 			// Is block expected?
 			bool block_expected (false);
-			bool unconfirmed_account_head (pull_blocks == 0 && !pull.confirmed_head && expected == pull.account_or_head && block->account () == pull.account_or_head);
+			bool unconfirmed_account_head (pull_blocks == 0 && pull.retry_limit != std::numeric_limits<unsigned>::max () && expected == pull.account_or_head && block->account () == pull.account_or_head);
 			if (hash == expected || unconfirmed_account_head)
 			{
 				expected = block->previous ();
@@ -235,7 +235,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e
 				connection->start_time = std::chrono::steady_clock::now ();
 			}
 			connection->attempt->total_blocks++;
-			bool stop_pull (connection->attempt->process_block (block, known_account, pull_blocks, block_expected, pull.confirmed_head));
+			bool stop_pull (connection->attempt->process_block (block, known_account, pull_blocks, pull.count, block_expected, pull.retry_limit));
 			pull_blocks++;
 			if (!stop_pull && !connection->hard_stop.load ())
 			{

nano/node/bootstrap/bootstrap_bulk_pull.hpp

@@ -12,15 +12,15 @@ class pull_info
 public:
 	using count_t = nano::bulk_pull::count_t;
 	pull_info () = default;
-	pull_info (nano::hash_or_account const &, nano::block_hash const &, nano::block_hash const &, count_t = 0, bool = false);
+	pull_info (nano::hash_or_account const &, nano::block_hash const &, nano::block_hash const &, count_t = 0, unsigned = 16);
 	nano::hash_or_account account_or_head{ 0 };
 	nano::block_hash head{ 0 };
 	nano::block_hash head_original{ 0 };
 	nano::block_hash end{ 0 };
 	count_t count{ 0 };
 	unsigned attempts{ 0 };
 	uint64_t processed{ 0 };
-	bool confirmed_head{ false };
+	unsigned retry_limit{ 0 };
 };
 class bootstrap_client;
 class bulk_pull_client final : public std::enable_shared_from_this<nano::bulk_pull_client>

nano/node/bootstrap/bootstrap_frontier.cpp

@@ -144,7 +144,7 @@ void nano::frontier_req_client::received_frontier (boost::system::error_code con
 						}
 						else
 						{
-							connection->attempt->add_pull (nano::pull_info (account, latest, frontier));
+							connection->attempt->add_pull (nano::pull_info (account, latest, frontier, 0, connection->node->network_params.bootstrap.frontier_retry_limit));
 							// Either we're behind or there's a fork we differ on
 							// Either way, bulk pushing will probably not be effective
 							bulk_push_cost += 5;
@@ -155,12 +155,12 @@ void nano::frontier_req_client::received_frontier (boost::system::error_code con
 				else
 				{
 					assert (account < current);
-					connection->attempt->add_pull (nano::pull_info (account, latest, nano::block_hash (0)));
+					connection->attempt->add_pull (nano::pull_info (account, latest, nano::block_hash (0), 0, connection->node->network_params.bootstrap.frontier_retry_limit));
 				}
 			}
 			else
 			{
-				connection->attempt->add_pull (nano::pull_info (account, latest, nano::block_hash (0)));
+				connection->attempt->add_pull (nano::pull_info (account, latest, nano::block_hash (0), 0, connection->node->network_params.bootstrap.frontier_retry_limit));
 			}
 			receive_frontier ();
 		}

nano/node/node.cpp

@@ -901,8 +901,10 @@ void nano::node::bootstrap_wallet ()
 void nano::node::unchecked_cleanup ()
 {
 	std::deque<nano::unchecked_key> cleaning_list;
+	auto attempt (bootstrap_initiator.current_attempt ());
+	bool long_attempt (attempt != nullptr && std::chrono::duration_cast<std::chrono::seconds> (std::chrono::steady_clock::now () - attempt->attempt_start).count () > config.unchecked_cutoff_time.count ());
 	// Collect old unchecked keys
-	if (!flags.disable_unchecked_cleanup && ledger.block_count_cache >= ledger.bootstrap_weight_max_blocks)
+	if (!flags.disable_unchecked_cleanup && ledger.block_count_cache >= ledger.bootstrap_weight_max_blocks && !long_attempt)
 	{
 		auto now (nano::seconds_since_epoch ());
 		auto transaction (store.tx_begin_read ());

nano/secure/common.cpp

@@ -142,6 +142,10 @@ nano::portmapping_constants::portmapping_constants (nano::network_constants & ne
 nano::bootstrap_constants::bootstrap_constants (nano::network_constants & network_constants)
 {
 	lazy_max_pull_blocks = network_constants.is_test_network () ? 2 : 512;
+	lazy_min_pull_blocks = network_constants.is_test_network () ? 1 : 32;
+	frontier_retry_limit = network_constants.is_test_network () ? 2 : 16;
+	lazy_retry_limit = network_constants.is_test_network () ? 2 : frontier_retry_limit * 10;
+	lazy_destinations_retry_limit = network_constants.is_test_network () ? 1 : frontier_retry_limit / 4;
 }
 
 /* Convenience constants for core_test which is always on the test network */

nano/secure/common.hpp

@@ -423,7 +423,11 @@ class bootstrap_constants
 {
 public:
 	bootstrap_constants (nano::network_constants & network_constants);
-	uint64_t lazy_max_pull_blocks;
+	uint32_t lazy_max_pull_blocks;
+	uint32_t lazy_min_pull_blocks;
+	unsigned frontier_retry_limit;
+	unsigned lazy_retry_limit;
+	unsigned lazy_destinations_retry_limit;
 };
 
 /** Constants whose value depends on the active network */