Merge pull request #3828 from clemahieu/v23.1

V23.1 Cumulative patch

nano/core_test/node.cpp

@@ -11,6 +11,7 @@
 #include <boost/make_shared.hpp>
 #include <boost/variant.hpp>
 
+#include <fstream>
 #include <numeric>
 
 using namespace std::chrono_literals;
@@ -4836,6 +4837,33 @@ TEST (node, pruning_depth)
 	ASSERT_TRUE (node1.ledger.block_or_pruned_exists (send2->hash ()));
 }
 
+TEST (node_config, node_id_private_key_persistence)
+{
+	nano::logger_mt logger;
+
+	// create the directory and the file
+	auto path = nano::unique_path ();
+	ASSERT_TRUE (boost::filesystem::create_directories (path));
+	auto priv_key_filename = path / "node_id_private.key";
+
+	// check that the key generated is random when the key does not exist
+	nano::keypair kp1 = nano::load_or_create_node_id (path, logger);
+	boost::filesystem::remove (priv_key_filename);
+	nano::keypair kp2 = nano::load_or_create_node_id (path, logger);
+	ASSERT_NE (kp1.prv, kp2.prv);
+
+	// check that the key persists
+	nano::keypair kp3 = nano::load_or_create_node_id (path, logger);
+	ASSERT_EQ (kp2.prv, kp3.prv);
+
+	// write the key file manually and check that right key is loaded
+	std::ofstream ofs (priv_key_filename.string (), std::ofstream::out | std::ofstream::trunc);
+	ofs << "3F28D035B8AA75EA53DF753BFD065CF6138E742971B2C99B84FD8FE328FED2D9" << std::flush;
+	ofs.close ();
+	nano::keypair kp4 = nano::load_or_create_node_id (path, logger);
+	ASSERT_EQ (kp4.prv, nano::keypair ("3F28D035B8AA75EA53DF753BFD065CF6138E742971B2C99B84FD8FE328FED2D9").prv);
+}
+
 namespace
 {
 void add_required_children_node_config_tree (nano::jsonconfig & tree)

nano/node/active_transactions.cpp

@@ -571,14 +571,6 @@ void nano::active_transactions::request_loop ()
 
 	while (!stopped && !node.flags.disable_request_loop)
 	{
-		// If many votes are queued, ensure at least the currently active ones finish processing
-		lock.unlock ();
-		if (node.vote_processor.half_full ())
-		{
-			node.vote_processor.flush_active ();
-		}
-		lock.lock ();
-
 		auto const stamp_l = std::chrono::steady_clock::now ();
 
 		request_confirm (lock);
@@ -1126,6 +1118,11 @@ std::size_t nano::active_transactions::inactive_votes_cache_size ()
 
 void nano::active_transactions::add_inactive_votes_cache (nano::unique_lock<nano::mutex> & lock_a, nano::block_hash const & hash_a, nano::account const & representative_a, uint64_t const timestamp_a)
 {
+	if (node.flags.inactive_votes_cache_size == 0)
+	{
+		return;
+	}
+
 	// Check principal representative status
 	if (node.ledger.weight (representative_a) > node.minimum_principal_weight ())
 	{

nano/node/blockprocessor.cpp

@@ -383,6 +383,10 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 			}
 
 			nano::unchecked_key unchecked_key (block->previous (), hash);
+			if (node.ledger.bootstrap_weight_reached () && node.store.unchecked.count (transaction_a) > max)
+			{
+				node.store.unchecked.clear (transaction_a);
+			}
 			node.store.unchecked.put (transaction_a, unchecked_key, info_a);
 
 			events_a.events.emplace_back ([this, hash] (nano::transaction const & /* unused */) { this->node.gap_cache.add (hash); });
@@ -403,6 +407,10 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 			}
 
 			nano::unchecked_key unchecked_key (node.ledger.block_source (transaction_a, *(block)), hash);
+			if (node.ledger.bootstrap_weight_reached () && node.store.unchecked.count (transaction_a) > max)
+			{
+				node.store.unchecked.clear (transaction_a);
+			}
 			node.store.unchecked.put (transaction_a, unchecked_key, info_a);
 
 			events_a.events.emplace_back ([this, hash] (nano::transaction const & /* unused */) { this->node.gap_cache.add (hash); });
@@ -423,6 +431,10 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 			}
 
 			nano::unchecked_key unchecked_key (block->account (), hash); // Specific unchecked key starting with epoch open block account public key
+			if (node.ledger.bootstrap_weight_reached () && node.store.unchecked.count (transaction_a) > max)
+			{
+				node.store.unchecked.clear (transaction_a);
+			}
 			node.store.unchecked.put (transaction_a, unchecked_key, info_a);
 
 			node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source);
@@ -474,7 +486,10 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 		}
 		case nano::process_result::opened_burn_account:
 		{
-			node.logger.always_log (boost::str (boost::format ("*** Rejecting open block for burn account ***: %1%") % hash.to_string ()));
+			if (node.config.logging.ledger_logging ())
+			{
+				node.logger.try_log (boost::str (boost::format ("Rejecting open block for burn account: %1%") % hash.to_string ()));
+			}
 			break;
 		}
 		case nano::process_result::balance_mismatch:

nano/node/blockprocessor.hpp

@@ -87,6 +87,7 @@ class block_processor final
 	nano::mutex mutex{ mutex_identifier (mutexes::block_processor) };
 	nano::state_block_signature_verification state_block_signature_verification;
 	std::thread processing_thread;
+	static uint64_t constexpr max = 256 * 1024;
 
 	friend std::unique_ptr<container_info_component> collect_container_info (block_processor & block_processor, std::string const & name);
 };

nano/node/bootstrap/bootstrap_server.cpp

@@ -662,10 +662,24 @@ class request_response_visitor : public nano::message_visitor
 	}
 	void node_id_handshake (nano::node_id_handshake const & message_a) override
 	{
+		// check for multiple handshake messages, there is no reason to receive more than one
+		if (message_a.query && connection->handshake_query_received)
+		{
+			if (connection->node->config.logging.network_node_id_handshake_logging ())
+			{
+				connection->node->logger.try_log (boost::str (boost::format ("Detected multiple node_id_handshake query from %1%") % connection->remote_endpoint));
+			}
+			connection->stop ();
+			return;
+		}
+
+		connection->handshake_query_received = true;
+
 		if (connection->node->config.logging.network_node_id_handshake_logging ())
 		{
 			connection->node->logger.try_log (boost::str (boost::format ("Received node_id_handshake message from %1%") % connection->remote_endpoint));
 		}
+
 		if (message_a.query)
 		{
 			boost::optional<std::pair<nano::account, nano::signature>> response (std::make_pair (connection->node->node_id.pub, nano::sign_message (connection->node->node_id.prv, connection->node->node_id.pub, *message_a.query)));

nano/node/bootstrap/bootstrap_server.hpp

@@ -64,6 +64,7 @@ class bootstrap_server final : public std::enable_shared_from_this<nano::bootstr
 	nano::mutex mutex;
 	std::queue<std::unique_ptr<nano::message>> requests;
 	std::atomic<bool> stopped{ false };
+	std::atomic<bool> handshake_query_received{ false };
 	// Remote enpoint used to remove response channel even after socket closing
 	nano::tcp_endpoint remote_endpoint{ boost::asio::ip::address_v6::any (), 0 };
 	nano::account remote_node_id{};

nano/node/node.cpp

@@ -16,6 +16,7 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <fstream>
 #include <future>
 #include <sstream>
 
@@ -74,6 +75,33 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (re
 	return composite;
 }
 
+nano::keypair nano::load_or_create_node_id (boost::filesystem::path const & application_path, nano::logger_mt & logger)
+{
+	auto node_private_key_path = application_path / "node_id_private.key";
+	std::ifstream ifs (node_private_key_path.c_str ());
+	if (ifs.good ())
+	{
+		logger.always_log (boost::str (boost::format ("%1% exists, reading node id from it") % node_private_key_path.string ()));
+		std::string node_private_key;
+		ifs >> node_private_key;
+		release_assert (node_private_key.size () == 64);
+		nano::keypair kp = nano::keypair (node_private_key);
+		return kp;
+	}
+	else
+	{
+		// no node_id found, generate new one
+		logger.always_log (boost::str (boost::format ("%1% does not exist, creating a new node_id") % node_private_key_path.string ()));
+		nano::keypair kp;
+		std::ofstream ofs (node_private_key_path.c_str (), std::ofstream::out | std::ofstream::trunc);
+		ofs << kp.prv.to_string () << std::endl
+			<< std::flush;
+		ofs.close ();
+		release_assert (!ofs.fail ());
+		return kp;
+	}
+}
+
 nano::node::node (boost::asio::io_context & io_ctx_a, uint16_t peering_port_a, boost::filesystem::path const & application_path_a, nano::logging const & logging_a, nano::work_pool & work_a, nano::node_flags flags_a, unsigned seq) :
 	node (io_ctx_a, application_path_a, nano::node_config (peering_port_a, logging_a), work_a, flags_a, seq)
 {
@@ -381,7 +409,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
 			logger.always_log (stream.str ());
 		}
 
-		node_id = nano::keypair ();
+		node_id = nano::load_or_create_node_id (application_path, logger);
 		logger.always_log ("Node ID: ", node_id.pub.to_node_id ());
 
 		if ((network_params.network.is_live_network () || network_params.network.is_beta_network ()) && !flags.inactive_node)

nano/node/node.hpp

@@ -213,6 +213,7 @@ class node final : public std::enable_shared_from_this<nano::node>
 	nano::locked<std::future<void>> epoch_upgrading;
 };
 
+nano::keypair load_or_create_node_id (boost::filesystem::path const & application_path, nano::logger_mt & logger);
 std::unique_ptr<container_info_component> collect_container_info (node & node, std::string const & name);
 
 nano::node_flags const & inactive_node_flag_defaults ();

nano/node/nodeconfig.hpp

@@ -157,7 +157,7 @@ class node_flags final
 	std::size_t block_processor_batch_size{ 0 };
 	std::size_t block_processor_full_size{ 65536 };
 	std::size_t block_processor_verification_size{ 0 };
-	std::size_t inactive_votes_cache_size{ 16 * 1024 };
+	std::size_t inactive_votes_cache_size{ 0 };
 	std::size_t vote_processor_capacity{ 144 * 1024 };
 	std::size_t bootstrap_interval{ 0 }; // For testing only
 };

nano/secure/ledger.cpp

@@ -1546,6 +1546,11 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
 	return error;
 }
 
+bool nano::ledger::bootstrap_weight_reached () const
+{
+	return cache.block_count >= bootstrap_weight_max_blocks;
+}
+
 nano::uncemented_info::uncemented_info (nano::block_hash const & cemented_frontier, nano::block_hash const & frontier, nano::account const & account) :
 	cemented_frontier (cemented_frontier), frontier (frontier), account (account)
 {

nano/secure/ledger.hpp

@@ -67,6 +67,7 @@ class ledger final
 	nano::link const & epoch_link (nano::epoch) const;
 	std::multimap<uint64_t, uncemented_info, std::greater<>> unconfirmed_frontiers () const;
 	bool migrate_lmdb_to_rocksdb (boost::filesystem::path const &) const;
+	bool bootstrap_weight_reached () const;
 	static nano::uint128_t const unit;
 	nano::ledger_constants & constants;
 	nano::store & store;

nano/secure/store/unchecked_store_partial.hpp

@@ -36,6 +36,8 @@ class unchecked_store_partial : public unchecked_store
 
 	void put (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) override
 	{
+		if (get (transaction_a, key_a.previous).size () > 1)
+			return;
 		nano::db_val<Val> info (info_a);
 		auto status (store.put (transaction_a, tables::unchecked, key_a, info));
 		release_assert_success (store, status);