From aba559e84aca7cd8fd88f1598cc6682634f1bd29 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 19 Dec 2024 01:24:20 -0800 Subject: [PATCH 01/10] make QUIC tpu QOS parameters configurable --- bench-tps/src/cli.rs | 2 +- core/src/tpu.rs | 39 ++-------- core/src/validator.rs | 63 ++++++++++------ local-cluster/src/local_cluster.rs | 32 +++------ streamer/src/nonblocking/quic.rs | 8 --- streamer/src/nonblocking/stream_throttle.rs | 16 ++--- streamer/src/nonblocking/testing_utilities.rs | 10 +-- streamer/src/quic.rs | 24 ++++--- test-validator/src/lib.rs | 17 ++--- validator/src/admin_rpc_service.rs | 15 +--- validator/src/cli.rs | 72 +++++++++++++++++-- validator/src/main.rs | 48 +++++++++++-- 12 files changed, 203 insertions(+), 143 deletions(-) diff --git a/bench-tps/src/cli.rs b/bench-tps/src/cli.rs index 4525ee5ca12c99..3c5d8e642ac1bd 100644 --- a/bench-tps/src/cli.rs +++ b/bench-tps/src/cli.rs @@ -11,7 +11,7 @@ use { pubkey::Pubkey, signature::{read_keypair_file, Keypair}, }, - solana_streamer::nonblocking::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, + solana_streamer::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, solana_tpu_client::tpu_client::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC}, std::{ net::{IpAddr, Ipv4Addr}, diff --git a/core/src/tpu.rs b/core/src/tpu.rs index d715bb5c7b0534..2a03333ffb03a6 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -37,10 +37,7 @@ use { }, solana_sdk::{clock::Slot, pubkey::Pubkey, quic::NotifyKeyUpdate, signature::Keypair}, solana_streamer::{ - quic::{ - spawn_server_multi, QuicServerParams, SpawnServerResult, MAX_STAKED_CONNECTIONS, - MAX_UNSTAKED_CONNECTIONS, - }, + quic::{spawn_server_multi, QuicServerParams, SpawnServerResult}, streamer::StakedNodes, }, solana_turbine::broadcast_stage::{BroadcastStage, BroadcastStageType}, @@ -54,9 +51,6 @@ use { tokio::sync::mpsc::Sender as AsyncSender, }; -// allow multiple connections for NAT and any open/close overlap -pub const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; - pub struct TpuSockets { pub transactions: Vec, pub transaction_forwards: Vec, @@ -115,7 +109,9 @@ impl Tpu { banking_tracer: Arc, tracer_thread_hdl: TracerThread, tpu_enable_udp: bool, - tpu_max_connections_per_ipaddr_per_minute: u64, + tpu_quic_server_config: QuicServerParams, + tpu_fwd_quic_server_config: QuicServerParams, + voe_quic_server_config: QuicServerParams, prioritization_fee_cache: &Arc, block_production_method: BlockProductionMethod, enable_block_production_forwarding: bool, @@ -178,15 +174,7 @@ impl Tpu { vote_packet_sender.clone(), exit.clone(), staked_nodes.clone(), - QuicServerParams { - max_connections_per_peer: 1, - max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, - coalesce: tpu_coalesce, - max_staked_connections: MAX_STAKED_CONNECTIONS - .saturating_add(MAX_UNSTAKED_CONNECTIONS), - max_unstaked_connections: 0, - ..QuicServerParams::default() - }, + voe_quic_server_config, ) .unwrap(); @@ -203,12 +191,7 @@ impl Tpu { packet_sender, exit.clone(), staked_nodes.clone(), - QuicServerParams { - max_connections_per_peer: MAX_QUIC_CONNECTIONS_PER_PEER, - max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, - coalesce: tpu_coalesce, - ..QuicServerParams::default() - }, + tpu_quic_server_config, ) .unwrap(); @@ -225,15 +208,7 @@ impl Tpu { forwarded_packet_sender, exit.clone(), staked_nodes.clone(), - QuicServerParams { - max_connections_per_peer: MAX_QUIC_CONNECTIONS_PER_PEER, - max_staked_connections: MAX_STAKED_CONNECTIONS - .saturating_add(MAX_UNSTAKED_CONNECTIONS), - max_unstaked_connections: 0, // Prevent unstaked nodes from forwarding transactions - max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, - coalesce: tpu_coalesce, - ..QuicServerParams::default() - }, + tpu_fwd_quic_server_config, ) .unwrap(); diff --git a/core/src/validator.rs b/core/src/validator.rs index 802dcc5fefdc2b..75e35f3d11927c 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -124,7 +124,10 @@ use { timing::timestamp, }, solana_send_transaction_service::send_transaction_service, - solana_streamer::{socket::SocketAddrSpace, streamer::StakedNodes}, + solana_streamer::{quic::QuicServerParams, socket::SocketAddrSpace, streamer::StakedNodes}, + solana_tpu_client::tpu_client::{ + DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC, DEFAULT_VOTE_USE_QUIC, + }, solana_turbine::{self, broadcast_stage::BroadcastStageType}, solana_unified_scheduler_pool::DefaultSchedulerPool, solana_vote_program::vote_state, @@ -483,8 +486,33 @@ pub struct ValidatorTpuConfig { pub tpu_connection_pool_size: usize, /// Controls if to enable UDP for TPU tansactions. pub tpu_enable_udp: bool, - /// Controls the new maximum connections per IpAddr per minute - pub tpu_max_connections_per_ipaddr_per_minute: u64, + /// QUIC server config for regular TPU + pub tpu_quic_server_config: QuicServerParams, + /// QUIC server config for TPU forward + pub tpu_fwd_quic_server_config: QuicServerParams, + /// QUIC server config for Vote + pub vote_quic_server_config: QuicServerParams, +} + +pub fn build_validator_tpu_config_for_test(tpu_enable_udp: bool) -> ValidatorTpuConfig { + let mut tpu_quic_server_config = QuicServerParams::default(); + tpu_quic_server_config.max_connections_per_peer = 32; // max connections per IpAddr per minute + + let mut tpu_fwd_quic_server_config = QuicServerParams::default(); + tpu_fwd_quic_server_config.max_connections_per_peer = 32; // max connections per IpAddr per minute + + let mut vote_quic_server_config = QuicServerParams::default(); + vote_quic_server_config.max_connections_per_peer = 32; // max connections per IpAddr per minute + + ValidatorTpuConfig { + use_quic: DEFAULT_TPU_USE_QUIC, + vote_use_quic: DEFAULT_VOTE_USE_QUIC, + tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, + tpu_enable_udp, + tpu_quic_server_config, + tpu_fwd_quic_server_config, + vote_quic_server_config, + } } pub struct Validator { @@ -546,7 +574,9 @@ impl Validator { vote_use_quic, tpu_connection_pool_size, tpu_enable_udp, - tpu_max_connections_per_ipaddr_per_minute, + tpu_quic_server_config, + tpu_fwd_quic_server_config, + vote_quic_server_config, } = tpu_config; let start_time = Instant::now(); @@ -1521,7 +1551,9 @@ impl Validator { banking_tracer, tracer_thread, tpu_enable_udp, - tpu_max_connections_per_ipaddr_per_minute, + tpu_quic_server_config, + tpu_fwd_quic_server_config, + vote_quic_server_config, &prioritization_fee_cache, config.block_production_method.clone(), config.enable_block_production_forwarding, @@ -2723,10 +2755,7 @@ mod tests { get_tmp_ledger_path_auto_delete, }, solana_sdk::{genesis_config::create_genesis_config, poh_config::PohConfig}, - solana_tpu_client::tpu_client::{ - DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, - DEFAULT_VOTE_USE_QUIC, - }, + solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, std::{fs::remove_dir_all, thread, time::Duration}, }; @@ -2764,13 +2793,7 @@ mod tests { None, // rpc_to_plugin_manager_receiver start_progress.clone(), SocketAddrSpace::Unspecified, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute for test - }, + build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); @@ -2986,13 +3009,7 @@ mod tests { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), SocketAddrSpace::Unspecified, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute for test - }, + build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start") diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index ce3d82dcddfe84..759b3a2664c96e 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -11,7 +11,9 @@ use { solana_client::connection_cache::ConnectionCache, solana_core::{ consensus::tower_storage::FileTowerStorage, - validator::{Validator, ValidatorConfig, ValidatorStartProgress, ValidatorTpuConfig}, + validator::{ + build_validator_tpu_config_for_test, Validator, ValidatorConfig, ValidatorStartProgress, + }, }, solana_gossip::{ cluster_info::Node, @@ -341,15 +343,9 @@ impl LocalCluster { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), socket_addr_space, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - // We are turning tpu_enable_udp to true in order to prevent concurrent local cluster tests - // to use the same QUIC ports due to SO_REUSEPORT. - tpu_enable_udp: true, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute - }, + // We are turning tpu_enable_udp to true in order to prevent concurrent local cluster tests + // to use the same QUIC ports due to SO_REUSEPORT. + build_validator_tpu_config_for_test(true), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); @@ -553,13 +549,7 @@ impl LocalCluster { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), socket_addr_space, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per mintute - }, + build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); @@ -1089,13 +1079,7 @@ impl Cluster for LocalCluster { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), socket_addr_space, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute, use higher value because of tests - }, + build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index aca98bb4b45996..7b5d6ae6143b08 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -87,14 +87,6 @@ const CONNECTION_CLOSE_REASON_TOO_MANY: &[u8] = b"too_many"; const CONNECTION_CLOSE_CODE_INVALID_STREAM: u32 = 5; const CONNECTION_CLOSE_REASON_INVALID_STREAM: &[u8] = b"invalid_stream"; -/// Limit to 250K PPS -pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250; - -/// The new connections per minute from a particular IP address. -/// Heuristically set to the default maximum concurrent connections -/// per IP address. Might be adjusted later. -pub const DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE: u64 = 8; - /// Total new connection counts per second. Heuristically taken from /// the default staked and unstaked connection limits. Might be adjusted /// later. diff --git a/streamer/src/nonblocking/stream_throttle.rs b/streamer/src/nonblocking/stream_throttle.rs index a9db82874b9c21..7baeab7d5b1646 100644 --- a/streamer/src/nonblocking/stream_throttle.rs +++ b/streamer/src/nonblocking/stream_throttle.rs @@ -236,10 +236,8 @@ pub mod test { use { super::*, crate::{ - nonblocking::{ - quic::DEFAULT_MAX_STREAMS_PER_MS, stream_throttle::STREAM_LOAD_EMA_INTERVAL_MS, - }, - quic::{StreamerStats, MAX_UNSTAKED_CONNECTIONS}, + nonblocking::stream_throttle::STREAM_LOAD_EMA_INTERVAL_MS, + quic::{StreamerStats, DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_MAX_UNSTAKED_CONNECTIONS}, }, std::{ sync::{atomic::Ordering, Arc}, @@ -251,7 +249,7 @@ pub mod test { fn test_max_streams_for_unstaked_connection() { let load_ema = Arc::new(StakedStreamLoadEMA::new( Arc::new(StreamerStats::default()), - MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, )); // 25K packets per ms * 20% / 500 max unstaked connections @@ -268,7 +266,7 @@ pub mod test { fn test_max_streams_for_staked_connection() { let load_ema = Arc::new(StakedStreamLoadEMA::new( Arc::new(StreamerStats::default()), - MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, )); @@ -448,7 +446,7 @@ pub mod test { fn test_update_ema() { let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( Arc::new(StreamerStats::default()), - MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, )); stream_load_ema @@ -477,7 +475,7 @@ pub mod test { fn test_update_ema_missing_interval() { let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( Arc::new(StreamerStats::default()), - MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, )); stream_load_ema @@ -497,7 +495,7 @@ pub mod test { fn test_update_ema_if_needed() { let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( Arc::new(StreamerStats::default()), - MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, )); stream_load_ema diff --git a/streamer/src/nonblocking/testing_utilities.rs b/streamer/src/nonblocking/testing_utilities.rs index 78adfd3171b52c..0e74f28b190551 100644 --- a/streamer/src/nonblocking/testing_utilities.rs +++ b/streamer/src/nonblocking/testing_utilities.rs @@ -2,13 +2,13 @@ use { super::quic::{ spawn_server_multi, SpawnNonBlockingServerResult, ALPN_TPU_PROTOCOL_ID, - DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, }, crate::{ quic::{ - QuicServerParams, StreamerStats, DEFAULT_TPU_COALESCE, MAX_STAKED_CONNECTIONS, - MAX_UNSTAKED_CONNECTIONS, + QuicServerParams, StreamerStats, DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, + DEFAULT_MAX_STAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_TPU_COALESCE, }, streamer::StakedNodes, }, @@ -64,8 +64,8 @@ impl Default for TestServerConfig { fn default() -> Self { Self { max_connections_per_peer: 1, - max_staked_connections: MAX_STAKED_CONNECTIONS, - max_unstaked_connections: MAX_UNSTAKED_CONNECTIONS, + max_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS, + max_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS, max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS, max_connections_per_ipaddr_per_min: DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, } diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 4a290d17c43714..c1a095a9038abf 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -1,9 +1,6 @@ use { crate::{ - nonblocking::quic::{ - ALPN_TPU_PROTOCOL_ID, DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, - DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, - }, + nonblocking::quic::{ALPN_TPU_PROTOCOL_ID, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT}, streamer::StakedNodes, }, crossbeam_channel::Sender, @@ -32,8 +29,19 @@ use { tokio::runtime::Runtime, }; -pub const MAX_STAKED_CONNECTIONS: usize = 2000; -pub const MAX_UNSTAKED_CONNECTIONS: usize = 500; +// allow multiple connections for NAT and any open/close overlap +pub const DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; + +pub const DEFAULT_MAX_STAKED_CONNECTIONS: usize = 2000; +pub const DEFAULT_MAX_UNSTAKED_CONNECTIONS: usize = 500; + +/// Limit to 250K PPS +pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250; + +/// The new connections per minute from a particular IP address. +/// Heuristically set to the default maximum concurrent connections +/// per IP address. Might be adjusted later. +pub const DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE: u64 = 8; // This will be adjusted and parameterized in follow-on PRs. pub const DEFAULT_QUIC_ENDPOINTS: usize = 1; @@ -579,8 +587,8 @@ impl Default for QuicServerParams { fn default() -> Self { QuicServerParams { max_connections_per_peer: 1, - max_staked_connections: MAX_STAKED_CONNECTIONS, - max_unstaked_connections: MAX_UNSTAKED_CONNECTIONS, + max_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS, + max_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS, max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS, max_connections_per_ipaddr_per_min: DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, wait_for_chunk_timeout: DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 66ea5a914647cc..274d4c5e69bf8b 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -13,7 +13,9 @@ use { solana_core::{ admin_rpc_post_init::AdminRpcRequestMetadataPostInit, consensus::tower_storage::TowerStorage, - validator::{Validator, ValidatorConfig, ValidatorStartProgress, ValidatorTpuConfig}, + validator::{ + build_validator_tpu_config_for_test, Validator, ValidatorConfig, ValidatorStartProgress, + }, }, solana_feature_set::FEATURE_NAMES, solana_geyser_plugin_manager::{ @@ -56,10 +58,7 @@ use { signature::{read_keypair_file, write_keypair_file, Keypair, Signer}, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::tpu_client::{ - DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, - DEFAULT_VOTE_USE_QUIC, - }, + solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, std::{ collections::{HashMap, HashSet}, ffi::OsStr, @@ -1046,13 +1045,7 @@ impl TestValidator { rpc_to_plugin_manager_receiver, config.start_progress.clone(), socket_addr_space, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp: config.tpu_enable_udp, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute for test - }, + build_validator_tpu_config_for_test(config.tpu_enable_udp), config.admin_rpc_service_post_init.clone(), )?); diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index cf23530d86e437..ca5bad8a4cea88 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -871,7 +871,7 @@ mod tests { }, solana_core::{ consensus::tower_storage::NullTowerStorage, - validator::{Validator, ValidatorConfig, ValidatorTpuConfig}, + validator::{build_validator_tpu_config_for_test, Validator, ValidatorConfig}, }, solana_gossip::cluster_info::{ClusterInfo, Node}, solana_inline_spl::token, @@ -893,10 +893,7 @@ mod tests { system_program, }, solana_streamer::socket::SocketAddrSpace, - solana_tpu_client::tpu_client::{ - DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, - DEFAULT_VOTE_USE_QUIC, - }, + solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, spl_token_2022::{ solana_program::{program_option::COption, program_pack::Pack}, state::{Account as TokenAccount, AccountState as TokenAccountState, Mint}, @@ -1398,13 +1395,7 @@ mod tests { None, // rpc_to_plugin_manager_receiver start_progress.clone(), SocketAddrSpace::Unspecified, - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute for test - }, + build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), post_init, ) .expect("assume successful validator start"); diff --git a/validator/src/cli.rs b/validator/src/cli.rs index a546379230605d..c7c73edc1f4a4c 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -47,17 +47,18 @@ use { solana_send_transaction_service::send_transaction_service::{ self, MAX_BATCH_SEND_RATE_MS, MAX_TRANSACTION_BATCH_SIZE, }, - solana_streamer::quic::DEFAULT_QUIC_ENDPOINTS, + solana_streamer::quic::{ + DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER, + DEFAULT_MAX_STAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS, + DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_QUIC_ENDPOINTS, + }, solana_tpu_client::tpu_client::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_VOTE_USE_QUIC}, solana_unified_scheduler_pool::DefaultSchedulerPool, std::{path::PathBuf, str::FromStr}, }; pub mod thread_args; -use { - solana_streamer::nonblocking::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, - thread_args::{thread_args, DefaultThreadArgs}, -}; +use thread_args::{thread_args, DefaultThreadArgs}; const EXCLUDE_KEY: &str = "account-index-exclude-key"; const INCLUDE_KEY: &str = "account-index-include-key"; @@ -906,6 +907,51 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .hidden(hidden_unless_forced()) .help("Controls if to use QUIC to send votes."), ) + .arg( + Arg::with_name("tpu_max_connections_per_peer") + .long("tpu-max-connections-per-peer") + .takes_value(true) + .default_value(&default_args.tpu_max_connections_per_peer) + .validator(is_parsable::) + .hidden(hidden_unless_forced()) + .help("Controls the max concurrent connections per IpAddr."), + ) + .arg( + Arg::with_name("max_tpu_staked_connections") + .long("max-tpu-staked-connections") + .takes_value(true) + .default_value(&default_args.max_tpu_staked_connections) + .validator(is_parsable::) + .hidden(hidden_unless_forced()) + .help("Controls the max concurrent connections for TPU from staked nodes."), + ) + .arg( + Arg::with_name("max_tpu_unstaked_connections") + .long("max-tpu-unstaked-connections") + .takes_value(true) + .default_value(&default_args.max_tpu_unstaked_connections) + .validator(is_parsable::) + .hidden(hidden_unless_forced()) + .help("Controls the max concurrent connections fort TPU from unstaked nodes."), + ) + .arg( + Arg::with_name("max_fwd_staked_connections") + .long("max-fwd-staked-connections") + .takes_value(true) + .default_value(&default_args.max_fwd_staked_connections) + .validator(is_parsable::) + .hidden(hidden_unless_forced()) + .help("Controls the max concurrent connections for TPU-forward from staked nodes."), + ) + .arg( + Arg::with_name("max_fwd_unstaked_connections") + .long("max-fwd-unstaked-connections") + .takes_value(true) + .default_value(&default_args.max_fwd_unstaked_connections) + .validator(is_parsable::) + .hidden(hidden_unless_forced()) + .help("Controls the max concurrent connections for TPU-forward from unstaked nodes."), + ) .arg( Arg::with_name("num_quic_endpoints") .long("num-quic-endpoints") @@ -2305,7 +2351,15 @@ pub struct DefaultArgs { pub accounts_shrink_optimize_total_space: String, pub accounts_shrink_ratio: String, pub tpu_connection_pool_size: String, + + pub tpu_max_connections_per_peer: String, pub tpu_max_connections_per_ipaddr_per_minute: String, + pub max_tpu_staked_connections: String, + pub max_tpu_unstaked_connections: String, + pub max_fwd_staked_connections: String, + pub max_fwd_unstaked_connections: String, + pub max_streams_per_ms: String, + pub num_quic_endpoints: String, pub vote_use_quic: String, @@ -2401,6 +2455,14 @@ impl DefaultArgs { tpu_max_connections_per_ipaddr_per_minute: DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE.to_string(), vote_use_quic: DEFAULT_VOTE_USE_QUIC.to_string(), + tpu_max_connections_per_peer: DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER.to_string(), + max_tpu_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS.to_string(), + max_tpu_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS.to_string(), + max_fwd_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS + .saturating_add(DEFAULT_MAX_UNSTAKED_CONNECTIONS) + .to_string(), + max_fwd_unstaked_connections: 0.to_string(), + max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS.to_string(), num_quic_endpoints: DEFAULT_QUIC_ENDPOINTS.to_string(), rpc_max_request_body_size: MAX_REQUEST_BODY_SIZE.to_string(), exit_min_idle_time: "10".to_string(), diff --git a/validator/src/main.rs b/validator/src/main.rs index 252f68689d4680..76fff9918d3d8b 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -73,7 +73,7 @@ use { signature::{read_keypair, Keypair, Signer}, }, solana_send_transaction_service::send_transaction_service, - solana_streamer::socket::SocketAddrSpace, + solana_streamer::{quic::QuicServerParams, socket::SocketAddrSpace}, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, std::{ collections::{HashSet, VecDeque}, @@ -1139,8 +1139,6 @@ pub fn main() { }; let tpu_connection_pool_size = value_t_or_exit!(matches, "tpu_connection_pool_size", usize); - let tpu_max_connections_per_ipaddr_per_minute = - value_t_or_exit!(matches, "tpu_max_connections_per_ipaddr_per_minute", u64); let shrink_ratio = value_t_or_exit!(matches, "accounts_shrink_ratio", f64); if !(0.0..=1.0).contains(&shrink_ratio) { @@ -1976,6 +1974,21 @@ pub fn main() { }); let num_quic_endpoints = value_t_or_exit!(matches, "num_quic_endpoints", NonZeroUsize); + + let tpu_max_connections_per_peer = + value_t_or_exit!(matches, "tpu_max_connections_per_peer", u64); + let max_tpu_staked_connections = value_t_or_exit!(matches, "max_tpu_staked_connections", u64); + let max_tpu_unstaked_connections = + value_t_or_exit!(matches, "max_tpu_unstaked_connections", u64); + + let max_fwd_staked_connections = value_t_or_exit!(matches, "max_fwd_staked_connections", u64); + let max_fwd_unstaked_connections = + value_t_or_exit!(matches, "max_fwd_unstaked_connections", u64); + + let tpu_max_connections_per_ipaddr_per_minute: u64 = + value_t_or_exit!(matches, "tpu_max_connections_per_ipaddr_per_minute", u64); + let max_streams_per_ms = value_t_or_exit!(matches, "max_streams_per_ms", u64); + let node_config = NodeConfig { gossip_addr, port_range: dynamic_port_range, @@ -2079,6 +2092,31 @@ pub fn main() { // the one pushed by bootstrap. node.info.hot_swap_pubkey(identity_keypair.pubkey()); + let tpu_quic_server_config = QuicServerParams { + max_connections_per_peer: tpu_max_connections_per_peer.try_into().unwrap(), + max_staked_connections: max_tpu_staked_connections.try_into().unwrap(), + max_unstaked_connections: max_tpu_unstaked_connections.try_into().unwrap(), + max_streams_per_ms, + max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, + coalesce: tpu_coalesce, + ..Default::default() + }; + + let tpu_fwd_quic_server_config = QuicServerParams { + max_connections_per_peer: tpu_max_connections_per_peer.try_into().unwrap(), + max_staked_connections: max_fwd_staked_connections.try_into().unwrap(), + max_unstaked_connections: max_fwd_unstaked_connections.try_into().unwrap(), + max_streams_per_ms, + max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, + coalesce: tpu_coalesce, + ..Default::default() + }; + + // Vote shares TPU forward's characteristics, except that we accept 1 connection + // per peer. + let mut vote_quic_server_config = tpu_fwd_quic_server_config.clone(); + vote_quic_server_config.max_connections_per_peer = 1; + let validator = match Validator::new( node, identity_keypair, @@ -2096,7 +2134,9 @@ pub fn main() { vote_use_quic, tpu_connection_pool_size, tpu_enable_udp, - tpu_max_connections_per_ipaddr_per_minute, + tpu_quic_server_config, + tpu_fwd_quic_server_config, + vote_quic_server_config, }, admin_service_post_init, ) { From c1621c90e3c8cb86c3b472ace892e9ea49e9e7da Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 19 Dec 2024 01:38:45 -0800 Subject: [PATCH 02/10] Use max_connections_per_ipaddr_per_min --- core/src/validator.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index 75e35f3d11927c..eaa824601380ad 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -496,13 +496,13 @@ pub struct ValidatorTpuConfig { pub fn build_validator_tpu_config_for_test(tpu_enable_udp: bool) -> ValidatorTpuConfig { let mut tpu_quic_server_config = QuicServerParams::default(); - tpu_quic_server_config.max_connections_per_peer = 32; // max connections per IpAddr per minute + tpu_quic_server_config.max_connections_per_ipaddr_per_min = 32; let mut tpu_fwd_quic_server_config = QuicServerParams::default(); - tpu_fwd_quic_server_config.max_connections_per_peer = 32; // max connections per IpAddr per minute + tpu_fwd_quic_server_config.max_connections_per_ipaddr_per_min = 32; let mut vote_quic_server_config = QuicServerParams::default(); - vote_quic_server_config.max_connections_per_peer = 32; // max connections per IpAddr per minute + vote_quic_server_config.max_connections_per_ipaddr_per_min = 32; ValidatorTpuConfig { use_quic: DEFAULT_TPU_USE_QUIC, From 803740422977bc584ca263f58573259d14a26e6b Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 19 Dec 2024 01:44:17 -0800 Subject: [PATCH 03/10] set max_unstaked_connections to 0 for tpu-fwd and vote in testing --- core/src/validator.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/validator.rs b/core/src/validator.rs index eaa824601380ad..8f1e4a0e190094 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -494,15 +494,19 @@ pub struct ValidatorTpuConfig { pub vote_quic_server_config: QuicServerParams, } +/// A convenient function to build a ValidatorTpuConfig for testing with good +/// default. pub fn build_validator_tpu_config_for_test(tpu_enable_udp: bool) -> ValidatorTpuConfig { let mut tpu_quic_server_config = QuicServerParams::default(); tpu_quic_server_config.max_connections_per_ipaddr_per_min = 32; let mut tpu_fwd_quic_server_config = QuicServerParams::default(); tpu_fwd_quic_server_config.max_connections_per_ipaddr_per_min = 32; + tpu_fwd_quic_server_config.max_unstaked_connections = 0; let mut vote_quic_server_config = QuicServerParams::default(); vote_quic_server_config.max_connections_per_ipaddr_per_min = 32; + vote_quic_server_config.max_unstaked_connections = 0; ValidatorTpuConfig { use_quic: DEFAULT_TPU_USE_QUIC, From 30267bc1d7fd626d1193713f56c7358b558da42c Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:55:52 -0800 Subject: [PATCH 04/10] fixed some clippy complaint --- core/src/validator.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index 8f1e4a0e190094..7aa598e6462c58 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -497,16 +497,19 @@ pub struct ValidatorTpuConfig { /// A convenient function to build a ValidatorTpuConfig for testing with good /// default. pub fn build_validator_tpu_config_for_test(tpu_enable_udp: bool) -> ValidatorTpuConfig { - let mut tpu_quic_server_config = QuicServerParams::default(); - tpu_quic_server_config.max_connections_per_ipaddr_per_min = 32; + let tpu_quic_server_config = QuicServerParams { + max_connections_per_ipaddr_per_min: 32, + ..Default::default() + }; - let mut tpu_fwd_quic_server_config = QuicServerParams::default(); - tpu_fwd_quic_server_config.max_connections_per_ipaddr_per_min = 32; - tpu_fwd_quic_server_config.max_unstaked_connections = 0; + let tpu_fwd_quic_server_config = QuicServerParams { + max_connections_per_ipaddr_per_min: 32, + max_unstaked_connections: 0, + ..Default::default() + }; - let mut vote_quic_server_config = QuicServerParams::default(); - vote_quic_server_config.max_connections_per_ipaddr_per_min = 32; - vote_quic_server_config.max_unstaked_connections = 0; + // vote and tpu_fwd share the same characteristics -- disallow non-staked connections: + let vote_quic_server_config = tpu_fwd_quic_server_config.clone(); ValidatorTpuConfig { use_quic: DEFAULT_TPU_USE_QUIC, From 9c91cef5e9f29e1c02178364c5d2f44019c04994 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:41:35 -0800 Subject: [PATCH 05/10] missing max-streams-per-ms --- validator/src/cli.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/validator/src/cli.rs b/validator/src/cli.rs index c7c73edc1f4a4c..cb9284f5f45ff5 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -952,6 +952,15 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .hidden(hidden_unless_forced()) .help("Controls the max concurrent connections for TPU-forward from unstaked nodes."), ) + .arg( + Arg::with_name("max_streams_per_ms") + .long("max-streams-per-ms") + .takes_value(true) + .default_value(&default_args.max_streams_per_ms) + .validator(is_parsable::) + .hidden(hidden_unless_forced()) + .help("Controls the max number of streams per steamer service."), + ) .arg( Arg::with_name("num_quic_endpoints") .long("num-quic-endpoints") From cefd3e6980feb7553820fa87b992c42dd06c4650 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:50:05 -0800 Subject: [PATCH 06/10] missing tpu_max_streams_per_ms --- validator/src/cli.rs | 14 +++++++------- validator/src/main.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/validator/src/cli.rs b/validator/src/cli.rs index cb9284f5f45ff5..da54096f71e669 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -953,14 +953,14 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .help("Controls the max concurrent connections for TPU-forward from unstaked nodes."), ) .arg( - Arg::with_name("max_streams_per_ms") - .long("max-streams-per-ms") + Arg::with_name("tpu_max_streams_per_ms") + .long("tpu-max-streams-per-ms") .takes_value(true) - .default_value(&default_args.max_streams_per_ms) + .default_value(&default_args.tpu_max_streams_per_ms) .validator(is_parsable::) .hidden(hidden_unless_forced()) - .help("Controls the max number of streams per steamer service."), - ) + .help("Controls the max number of streams for a TPU service."), + ) .arg( Arg::with_name("num_quic_endpoints") .long("num-quic-endpoints") @@ -2367,7 +2367,7 @@ pub struct DefaultArgs { pub max_tpu_unstaked_connections: String, pub max_fwd_staked_connections: String, pub max_fwd_unstaked_connections: String, - pub max_streams_per_ms: String, + pub tpu_max_streams_per_ms: String, pub num_quic_endpoints: String, pub vote_use_quic: String, @@ -2471,7 +2471,7 @@ impl DefaultArgs { .saturating_add(DEFAULT_MAX_UNSTAKED_CONNECTIONS) .to_string(), max_fwd_unstaked_connections: 0.to_string(), - max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS.to_string(), + tpu_max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS.to_string(), num_quic_endpoints: DEFAULT_QUIC_ENDPOINTS.to_string(), rpc_max_request_body_size: MAX_REQUEST_BODY_SIZE.to_string(), exit_min_idle_time: "10".to_string(), diff --git a/validator/src/main.rs b/validator/src/main.rs index 76fff9918d3d8b..6082d2af973614 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1987,7 +1987,7 @@ pub fn main() { let tpu_max_connections_per_ipaddr_per_minute: u64 = value_t_or_exit!(matches, "tpu_max_connections_per_ipaddr_per_minute", u64); - let max_streams_per_ms = value_t_or_exit!(matches, "max_streams_per_ms", u64); + let max_streams_per_ms = value_t_or_exit!(matches, "tpu_max_streams_per_ms", u64); let node_config = NodeConfig { gossip_addr, From 3e74ee7902555073b8ce35561e8199eadf6ef539 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Fri, 20 Dec 2024 01:37:46 -0800 Subject: [PATCH 07/10] vote does not accept unstaked connections --- validator/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator/src/main.rs b/validator/src/main.rs index 6082d2af973614..0b5bbd8278b44c 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -2113,9 +2113,10 @@ pub fn main() { }; // Vote shares TPU forward's characteristics, except that we accept 1 connection - // per peer. + // per peer and no unstaked connections are accepted. let mut vote_quic_server_config = tpu_fwd_quic_server_config.clone(); vote_quic_server_config.max_connections_per_peer = 1; + vote_quic_server_config.max_unstaked_connections = 0; let validator = match Validator::new( node, From a22ceba2ea2ed8e0e42626ae69c432d988fd0f72 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sun, 22 Dec 2024 15:01:48 -0800 Subject: [PATCH 08/10] Addressed some feedback from Alessandro --- core/src/tpu.rs | 4 +-- core/src/validator.rs | 52 ++++++++++++++++-------------- local-cluster/src/local_cluster.rs | 10 +++--- test-validator/src/lib.rs | 6 ++-- validator/src/admin_rpc_service.rs | 4 +-- validator/src/cli.rs | 40 +++++++++++------------ validator/src/main.rs | 21 ++++++------ vortexor/src/cli.rs | 16 ++++----- vortexor/tests/vortexor.rs | 16 ++++----- 9 files changed, 83 insertions(+), 86 deletions(-) diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 2a03333ffb03a6..877c527d99347b 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -111,7 +111,7 @@ impl Tpu { tpu_enable_udp: bool, tpu_quic_server_config: QuicServerParams, tpu_fwd_quic_server_config: QuicServerParams, - voe_quic_server_config: QuicServerParams, + vote_quic_server_config: QuicServerParams, prioritization_fee_cache: &Arc, block_production_method: BlockProductionMethod, enable_block_production_forwarding: bool, @@ -174,7 +174,7 @@ impl Tpu { vote_packet_sender.clone(), exit.clone(), staked_nodes.clone(), - voe_quic_server_config, + vote_quic_server_config, ) .unwrap(); diff --git a/core/src/validator.rs b/core/src/validator.rs index 7aa598e6462c58..3c6c532d7940ca 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -494,31 +494,33 @@ pub struct ValidatorTpuConfig { pub vote_quic_server_config: QuicServerParams, } -/// A convenient function to build a ValidatorTpuConfig for testing with good -/// default. -pub fn build_validator_tpu_config_for_test(tpu_enable_udp: bool) -> ValidatorTpuConfig { - let tpu_quic_server_config = QuicServerParams { - max_connections_per_ipaddr_per_min: 32, - ..Default::default() - }; +impl ValidatorTpuConfig { + /// A convenient function to build a ValidatorTpuConfig for testing with good + /// default. + pub fn new_for_tests(tpu_enable_udp: bool) -> Self { + let tpu_quic_server_config = QuicServerParams { + max_connections_per_ipaddr_per_min: 32, + ..Default::default() + }; - let tpu_fwd_quic_server_config = QuicServerParams { - max_connections_per_ipaddr_per_min: 32, - max_unstaked_connections: 0, - ..Default::default() - }; + let tpu_fwd_quic_server_config = QuicServerParams { + max_connections_per_ipaddr_per_min: 32, + max_unstaked_connections: 0, + ..Default::default() + }; + + // vote and tpu_fwd share the same characteristics -- disallow non-staked connections: + let vote_quic_server_config = tpu_fwd_quic_server_config.clone(); - // vote and tpu_fwd share the same characteristics -- disallow non-staked connections: - let vote_quic_server_config = tpu_fwd_quic_server_config.clone(); - - ValidatorTpuConfig { - use_quic: DEFAULT_TPU_USE_QUIC, - vote_use_quic: DEFAULT_VOTE_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - tpu_enable_udp, - tpu_quic_server_config, - tpu_fwd_quic_server_config, - vote_quic_server_config, + ValidatorTpuConfig { + use_quic: DEFAULT_TPU_USE_QUIC, + vote_use_quic: DEFAULT_VOTE_USE_QUIC, + tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, + tpu_enable_udp, + tpu_quic_server_config, + tpu_fwd_quic_server_config, + vote_quic_server_config, + } } } @@ -2800,7 +2802,7 @@ mod tests { None, // rpc_to_plugin_manager_receiver start_progress.clone(), SocketAddrSpace::Unspecified, - build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), + ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); @@ -3016,7 +3018,7 @@ mod tests { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), SocketAddrSpace::Unspecified, - build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), + ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start") diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index 759b3a2664c96e..56987fd22d19ec 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -11,9 +11,7 @@ use { solana_client::connection_cache::ConnectionCache, solana_core::{ consensus::tower_storage::FileTowerStorage, - validator::{ - build_validator_tpu_config_for_test, Validator, ValidatorConfig, ValidatorStartProgress, - }, + validator::{Validator, ValidatorConfig, ValidatorStartProgress, ValidatorTpuConfig}, }, solana_gossip::{ cluster_info::Node, @@ -345,7 +343,7 @@ impl LocalCluster { socket_addr_space, // We are turning tpu_enable_udp to true in order to prevent concurrent local cluster tests // to use the same QUIC ports due to SO_REUSEPORT. - build_validator_tpu_config_for_test(true), + ValidatorTpuConfig::new_for_tests(true), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); @@ -549,7 +547,7 @@ impl LocalCluster { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), socket_addr_space, - build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), + ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); @@ -1079,7 +1077,7 @@ impl Cluster for LocalCluster { None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), socket_addr_space, - build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), + ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP), Arc::new(RwLock::new(None)), ) .expect("assume successful validator start"); diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 274d4c5e69bf8b..75718c6ea8791f 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -13,9 +13,7 @@ use { solana_core::{ admin_rpc_post_init::AdminRpcRequestMetadataPostInit, consensus::tower_storage::TowerStorage, - validator::{ - build_validator_tpu_config_for_test, Validator, ValidatorConfig, ValidatorStartProgress, - }, + validator::{Validator, ValidatorConfig, ValidatorStartProgress, ValidatorTpuConfig}, }, solana_feature_set::FEATURE_NAMES, solana_geyser_plugin_manager::{ @@ -1045,7 +1043,7 @@ impl TestValidator { rpc_to_plugin_manager_receiver, config.start_progress.clone(), socket_addr_space, - build_validator_tpu_config_for_test(config.tpu_enable_udp), + ValidatorTpuConfig::new_for_tests(config.tpu_enable_udp), config.admin_rpc_service_post_init.clone(), )?); diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index ca5bad8a4cea88..ebf2def516b597 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -871,7 +871,7 @@ mod tests { }, solana_core::{ consensus::tower_storage::NullTowerStorage, - validator::{build_validator_tpu_config_for_test, Validator, ValidatorConfig}, + validator::{Validator, ValidatorConfig, ValidatorTpuConfig}, }, solana_gossip::cluster_info::{ClusterInfo, Node}, solana_inline_spl::token, @@ -1395,7 +1395,7 @@ mod tests { None, // rpc_to_plugin_manager_receiver start_progress.clone(), SocketAddrSpace::Unspecified, - build_validator_tpu_config_for_test(DEFAULT_TPU_ENABLE_UDP), + ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP), post_init, ) .expect("assume successful validator start"); diff --git a/validator/src/cli.rs b/validator/src/cli.rs index da54096f71e669..48086682e4a33c 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -917,37 +917,37 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .help("Controls the max concurrent connections per IpAddr."), ) .arg( - Arg::with_name("max_tpu_staked_connections") - .long("max-tpu-staked-connections") + Arg::with_name("tpu_max_staked_connections") + .long("tpu-max-staked-connections") .takes_value(true) - .default_value(&default_args.max_tpu_staked_connections) + .default_value(&default_args.tpu_max_staked_connections) .validator(is_parsable::) .hidden(hidden_unless_forced()) .help("Controls the max concurrent connections for TPU from staked nodes."), ) .arg( - Arg::with_name("max_tpu_unstaked_connections") - .long("max-tpu-unstaked-connections") + Arg::with_name("tpu_max_unstaked_connections") + .long("tpu-max-unstaked-connections") .takes_value(true) - .default_value(&default_args.max_tpu_unstaked_connections) + .default_value(&default_args.tpu_max_unstaked_connections) .validator(is_parsable::) .hidden(hidden_unless_forced()) .help("Controls the max concurrent connections fort TPU from unstaked nodes."), ) .arg( - Arg::with_name("max_fwd_staked_connections") - .long("max-fwd-staked-connections") + Arg::with_name("tpu_max_fwd_staked_connections") + .long("tpu-max-fwd-staked-connections") .takes_value(true) - .default_value(&default_args.max_fwd_staked_connections) + .default_value(&default_args.tpu_max_fwd_staked_connections) .validator(is_parsable::) .hidden(hidden_unless_forced()) .help("Controls the max concurrent connections for TPU-forward from staked nodes."), ) .arg( - Arg::with_name("max_fwd_unstaked_connections") - .long("max-fwd-unstaked-connections") + Arg::with_name("tpu_max_fwd_unstaked_connections") + .long("tpu-max-fwd-unstaked-connections") .takes_value(true) - .default_value(&default_args.max_fwd_unstaked_connections) + .default_value(&default_args.tpu_max_fwd_unstaked_connections) .validator(is_parsable::) .hidden(hidden_unless_forced()) .help("Controls the max concurrent connections for TPU-forward from unstaked nodes."), @@ -2363,10 +2363,10 @@ pub struct DefaultArgs { pub tpu_max_connections_per_peer: String, pub tpu_max_connections_per_ipaddr_per_minute: String, - pub max_tpu_staked_connections: String, - pub max_tpu_unstaked_connections: String, - pub max_fwd_staked_connections: String, - pub max_fwd_unstaked_connections: String, + pub tpu_max_staked_connections: String, + pub tpu_max_unstaked_connections: String, + pub tpu_max_fwd_staked_connections: String, + pub tpu_max_fwd_unstaked_connections: String, pub tpu_max_streams_per_ms: String, pub num_quic_endpoints: String, @@ -2465,12 +2465,12 @@ impl DefaultArgs { DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE.to_string(), vote_use_quic: DEFAULT_VOTE_USE_QUIC.to_string(), tpu_max_connections_per_peer: DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER.to_string(), - max_tpu_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS.to_string(), - max_tpu_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS.to_string(), - max_fwd_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS + tpu_max_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS.to_string(), + tpu_max_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS.to_string(), + tpu_max_fwd_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS .saturating_add(DEFAULT_MAX_UNSTAKED_CONNECTIONS) .to_string(), - max_fwd_unstaked_connections: 0.to_string(), + tpu_max_fwd_unstaked_connections: 0.to_string(), tpu_max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS.to_string(), num_quic_endpoints: DEFAULT_QUIC_ENDPOINTS.to_string(), rpc_max_request_body_size: MAX_REQUEST_BODY_SIZE.to_string(), diff --git a/validator/src/main.rs b/validator/src/main.rs index 0b5bbd8278b44c..64569c53f7d656 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1977,13 +1977,14 @@ pub fn main() { let tpu_max_connections_per_peer = value_t_or_exit!(matches, "tpu_max_connections_per_peer", u64); - let max_tpu_staked_connections = value_t_or_exit!(matches, "max_tpu_staked_connections", u64); - let max_tpu_unstaked_connections = - value_t_or_exit!(matches, "max_tpu_unstaked_connections", u64); + let tpu_max_staked_connections = value_t_or_exit!(matches, "tpu_max_staked_connections", u64); + let tpu_max_unstaked_connections = + value_t_or_exit!(matches, "tpu_max_unstaked_connections", u64); - let max_fwd_staked_connections = value_t_or_exit!(matches, "max_fwd_staked_connections", u64); - let max_fwd_unstaked_connections = - value_t_or_exit!(matches, "max_fwd_unstaked_connections", u64); + let tpu_max_fwd_staked_connections = + value_t_or_exit!(matches, "tpu_max_fwd_staked_connections", u64); + let tpu_max_fwd_unstaked_connections = + value_t_or_exit!(matches, "tpu_max_fwd_unstaked_connections", u64); let tpu_max_connections_per_ipaddr_per_minute: u64 = value_t_or_exit!(matches, "tpu_max_connections_per_ipaddr_per_minute", u64); @@ -2094,8 +2095,8 @@ pub fn main() { let tpu_quic_server_config = QuicServerParams { max_connections_per_peer: tpu_max_connections_per_peer.try_into().unwrap(), - max_staked_connections: max_tpu_staked_connections.try_into().unwrap(), - max_unstaked_connections: max_tpu_unstaked_connections.try_into().unwrap(), + max_staked_connections: tpu_max_staked_connections.try_into().unwrap(), + max_unstaked_connections: tpu_max_unstaked_connections.try_into().unwrap(), max_streams_per_ms, max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, coalesce: tpu_coalesce, @@ -2104,8 +2105,8 @@ pub fn main() { let tpu_fwd_quic_server_config = QuicServerParams { max_connections_per_peer: tpu_max_connections_per_peer.try_into().unwrap(), - max_staked_connections: max_fwd_staked_connections.try_into().unwrap(), - max_unstaked_connections: max_fwd_unstaked_connections.try_into().unwrap(), + max_staked_connections: tpu_max_fwd_staked_connections.try_into().unwrap(), + max_unstaked_connections: tpu_max_fwd_unstaked_connections.try_into().unwrap(), max_streams_per_ms, max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute, coalesce: tpu_coalesce, diff --git a/vortexor/src/cli.rs b/vortexor/src/cli.rs index 30df80952c6bad..02091fa659dc64 100644 --- a/vortexor/src/cli.rs +++ b/vortexor/src/cli.rs @@ -3,11 +3,9 @@ use { solana_clap_utils::input_validators::{is_keypair_or_ask_keyword, is_parsable}, solana_net_utils::{MINIMUM_VALIDATOR_PORT_RANGE_WIDTH, VALIDATOR_PORT_RANGE}, solana_sdk::quic::QUIC_PORT_OFFSET, - solana_streamer::{ - nonblocking::quic::{ - DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_STREAMS_PER_MS, - }, - quic::{MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, + solana_streamer::quic::{ + DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_STAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_MAX_UNSTAKED_CONNECTIONS, }, }; @@ -33,10 +31,10 @@ impl Default for DefaultArgs { bind_address: "0.0.0.0".to_string(), dynamic_port_range: format!("{}-{}", VALIDATOR_PORT_RANGE.0, VALIDATOR_PORT_RANGE.1), max_connections_per_peer: DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER.to_string(), - max_tpu_staked_connections: MAX_STAKED_CONNECTIONS.to_string(), - max_tpu_unstaked_connections: MAX_UNSTAKED_CONNECTIONS.to_string(), - max_fwd_staked_connections: MAX_STAKED_CONNECTIONS - .saturating_add(MAX_UNSTAKED_CONNECTIONS) + max_tpu_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS.to_string(), + max_tpu_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS.to_string(), + max_fwd_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS + .saturating_add(DEFAULT_MAX_UNSTAKED_CONNECTIONS) .to_string(), max_fwd_unstaked_connections: 0.to_string(), max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS.to_string(), diff --git a/vortexor/tests/vortexor.rs b/vortexor/tests/vortexor.rs index 8d2f683c9e2006..17ab106c7753e2 100644 --- a/vortexor/tests/vortexor.rs +++ b/vortexor/tests/vortexor.rs @@ -3,11 +3,11 @@ use { solana_net_utils::VALIDATOR_PORT_RANGE, solana_sdk::{net::DEFAULT_TPU_COALESCE, pubkey::Pubkey, signature::Keypair, signer::Signer}, solana_streamer::{ - nonblocking::{ - quic::{DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_STREAMS_PER_MS}, - testing_utilities::check_multiple_streams, + nonblocking::testing_utilities::check_multiple_streams, + quic::{ + DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_STAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_MAX_UNSTAKED_CONNECTIONS, }, - quic::{MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, streamer::StakedNodes, }, solana_vortexor::{ @@ -54,10 +54,10 @@ async fn test_vortexor() { tpu_sender, tpu_fwd_sender, DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER.try_into().unwrap(), - MAX_STAKED_CONNECTIONS.try_into().unwrap(), - MAX_UNSTAKED_CONNECTIONS.try_into().unwrap(), - MAX_STAKED_CONNECTIONS - .saturating_add(MAX_UNSTAKED_CONNECTIONS) + DEFAULT_MAX_STAKED_CONNECTIONS.try_into().unwrap(), + DEFAULT_MAX_UNSTAKED_CONNECTIONS.try_into().unwrap(), + DEFAULT_MAX_STAKED_CONNECTIONS + .saturating_add(DEFAULT_MAX_UNSTAKED_CONNECTIONS) .try_into() .unwrap(), // max_fwd_staked_connections 0, // max_fwd_unstaked_connections From 133db69de1ad85f663423b50457fa119b94c294c Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sat, 11 Jan 2025 12:45:20 -0800 Subject: [PATCH 09/10] re-export some constants moved/renamed and mark them deprecated --- core/src/tpu.rs | 7 +++++++ streamer/src/nonblocking/quic.rs | 16 ++++++++++++++++ streamer/src/quic.rs | 13 +++++++++++++ 3 files changed, 36 insertions(+) diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 877c527d99347b..01904564420196 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -51,6 +51,13 @@ use { tokio::sync::mpsc::Sender as AsyncSender, }; +// allow multiple connections for NAT and any open/close overlap +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER instead" +)] +pub const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; + pub struct TpuSockets { pub transactions: Vec, pub transaction_forwards: Vec, diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 7b5d6ae6143b08..4d879f939ad46c 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -87,6 +87,22 @@ const CONNECTION_CLOSE_REASON_TOO_MANY: &[u8] = b"too_many"; const CONNECTION_CLOSE_CODE_INVALID_STREAM: u32 = 5; const CONNECTION_CLOSE_REASON_INVALID_STREAM: &[u8] = b"invalid_stream"; +/// Limit to 250K PPS +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_STREAMS_PER_MS" +)] +pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250; + +/// The new connections per minute from a particular IP address. +/// Heuristically set to the default maximum concurrent connections +/// per IP address. Might be adjusted later. +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE" +)] +pub const DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE: u64 = 8; + /// Total new connection counts per second. Heuristically taken from /// the default staked and unstaked connection limits. Might be adjusted /// later. diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index c1a095a9038abf..b9c352c02e4883 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -32,7 +32,20 @@ use { // allow multiple connections for NAT and any open/close overlap pub const DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_STAKED_CONNECTIONS" +)] +pub const MAX_STAKED_CONNECTIONS: usize = 2000; + pub const DEFAULT_MAX_STAKED_CONNECTIONS: usize = 2000; + +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_UNSTAKED_CONNECTIONS" +)] +pub const MAX_UNSTAKED_CONNECTIONS: usize = 500; + pub const DEFAULT_MAX_UNSTAKED_CONNECTIONS: usize = 500; /// Limit to 250K PPS From 7aea7c0b68a6d74f0e2999df012dd6b9534101b9 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Tue, 14 Jan 2025 18:55:08 -0800 Subject: [PATCH 10/10] removed duplicated code definition, use 'use' --- core/src/tpu.rs | 13 ++++++------- streamer/src/nonblocking/quic.rs | 15 +++++++-------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 01904564420196..cdcd0066106bb9 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -2,6 +2,12 @@ //! multi-stage transaction processing pipeline in software. pub use solana_sdk::net::DEFAULT_TPU_COALESCE; +// allow multiple connections for NAT and any open/close overlap +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER instead" +)] +pub use solana_streamer::quic::DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER as MAX_QUIC_CONNECTIONS_PER_PEER; use { crate::{ banking_stage::BankingStage, @@ -51,13 +57,6 @@ use { tokio::sync::mpsc::Sender as AsyncSender, }; -// allow multiple connections for NAT and any open/close overlap -#[deprecated( - since = "2.2.0", - note = "Use solana_streamer::quic::DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER instead" -)] -pub const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; - pub struct TpuSockets { pub transactions: Vec, pub transaction_forwards: Vec, diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 4d879f939ad46c..3902f843321901 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -87,13 +87,6 @@ const CONNECTION_CLOSE_REASON_TOO_MANY: &[u8] = b"too_many"; const CONNECTION_CLOSE_CODE_INVALID_STREAM: u32 = 5; const CONNECTION_CLOSE_REASON_INVALID_STREAM: &[u8] = b"invalid_stream"; -/// Limit to 250K PPS -#[deprecated( - since = "2.2.0", - note = "Use solana_streamer::quic::DEFAULT_MAX_STREAMS_PER_MS" -)] -pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250; - /// The new connections per minute from a particular IP address. /// Heuristically set to the default maximum concurrent connections /// per IP address. Might be adjusted later. @@ -101,7 +94,13 @@ pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250; since = "2.2.0", note = "Use solana_streamer::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE" )] -pub const DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE: u64 = 8; +pub use crate::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE; +/// Limit to 250K PPS +#[deprecated( + since = "2.2.0", + note = "Use solana_streamer::quic::DEFAULT_MAX_STREAMS_PER_MS" +)] +pub use crate::quic::DEFAULT_MAX_STREAMS_PER_MS; /// Total new connection counts per second. Heuristically taken from /// the default staked and unstaked connection limits. Might be adjusted