Mihailjianu1/bitcoin crate upgrade+testnet4 all

ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh

@@ -69,11 +69,9 @@ if [ "${SOCKS_FILE}" != "" -a -e "${SOCKS_FILE}" ]; then
     read_socks_proxy "${SOCKS_FILE}"
 fi
 
-BITCOIN_NETWORK='"testnet"'
-DNS_SEEDS='"testnet-seed.bitcoin.jonasschnelli.ch",
-            "seed.tbtc.petertodd.org",
-            "seed.testnet.bitcoin.sprovoost.nl",
-            "testnet-seed.bluematt.me"'
+BITCOIN_NETWORK='"testnet4"'
+DNS_SEEDS='"seed.testnet4.bitcoin.sprovoost.nl",
+            "seed.testnet4.wiz.biz"'
 
 if [ "$MAINNET" = true ]; then
     BITCOIN_NETWORK='"bitcoin"'

rs/bitcoin/adapter/src/blockchainmanager.rs

@@ -430,7 +430,7 @@ impl BlockchainManager {
         };
 
         let time_taken = request.sent_at.map(|i| i.elapsed()).unwrap_or_default();
-        trace!(
+        info!(
             self.logger,
             "Received block message from {} : Took {:?}sec. Block {:?}",
             addr,
@@ -439,7 +439,10 @@ impl BlockchainManager {
         );
 
         match self.blockchain.lock().unwrap().add_block(block.clone()) {
-            Ok(()) => Ok(()),
+            Ok(()) => {
+                info!(self.logger, "Added block: {}", block_hash);
+                Ok(())
+            },
             Err(err) => {
                 warn!(
                     self.logger,
@@ -535,6 +538,13 @@ impl BlockchainManager {
             }
         }
 
+        // Remove the requests that have timed out.
+        // If they are not removed, then all peers could be fully saturated and we can't send any requests.
+        for block_hash in retry_queue.iter() {
+            info!(self.logger, "Retrying getdata request for block: {}", block_hash);
+            self.getdata_request_info.remove(block_hash);
+        }
+
         // If nothing to be synced, then there is nothing to do at this point.
         if retry_queue.is_empty() && self.block_sync_queue.is_empty() {
             return;
@@ -543,7 +553,7 @@ impl BlockchainManager {
         let block_cache_size = self.blockchain.lock().unwrap().get_block_cache_size();
 
         if block_cache_size >= BLOCK_CACHE_THRESHOLD_BYTES {
-            debug!(
+            info!(
                 self.logger,
                 "Cache Size: {}, Max Size: {}", block_cache_size, BLOCK_CACHE_THRESHOLD_BYTES
             );
@@ -567,7 +577,9 @@ impl BlockchainManager {
         });
 
         // For each peer, select a random subset of the inventory and send a "getdata" request for it.
+        info!(self.logger, "sync_blocks: retry queue size: {}, sync queue {}:", retry_queue.len(), self.block_sync_queue.len());
         for peer in peer_info {
+            info!(self.logger, "trying from peer: {}, requests: {}", peer.socket, requests_per_peer.get(&peer.socket).unwrap_or(&0));
             // Calculate number of inventory that can be sent in 'getdata' request to the peer.
             let requests_sent_to_peer = requests_per_peer.get(&peer.socket).unwrap_or(&0);
             let num_requests_to_be_sent =
@@ -589,10 +601,10 @@ impl BlockchainManager {
             }
 
             if selected_inventory.is_empty() {
-                break;
+                continue;
             }
 
-            trace!(
+            info!(
                 self.logger,
                 "Sending getdata to {} : Inventory {:?}",
                 peer.socket,
@@ -753,20 +765,20 @@ impl BlockchainManager {
     }
 }
 
+// Prioritizes elements in sync_queue (as elements in retry_queue are vety likely bad blocks).
 fn get_next_block_hash_to_sync(
     is_cache_full: bool,
     retry_queue: &mut LinkedHashSet<BlockHash>,
     sync_queue: &mut LinkedHashSet<BlockHash>,
 ) -> Option<BlockHash> {
-    if !retry_queue.is_empty() {
-        return retry_queue.pop_front();
-    }
-
     if is_cache_full {
         return None;
     }
 
-    sync_queue.pop_front()
+    if !sync_queue.is_empty() {
+        return sync_queue.pop_front();
+    }
+    retry_queue.pop_front()
 }
 
 #[cfg(test)]

rs/bitcoin/adapter/src/config.rs

@@ -62,9 +62,11 @@ pub(crate) fn address_limits(network: Network) -> (usize, usize) {
     match network {
         Network::Bitcoin => (500, 2000),
         Network::Testnet => (100, 1000),
+        //TODO(mihailjianu): revisit these values
+        Network::Testnet4 => (100, 1000),
         Network::Signet => (1, 1),
         Network::Regtest => (1, 1),
-        other => unreachable!("Unsupported network: {:?}", other),
+        _ => (500, 2000),
     }
 }
 
@@ -74,6 +76,7 @@ impl Config {
         match self.network {
             Network::Bitcoin => 8333,
             Network::Testnet => 18333,
+            Network::Testnet4 => 48333,
             _ => 8333,
         }
     }

rs/bitcoin/adapter/src/get_successors_handler.rs

@@ -19,7 +19,7 @@ use crate::{
 //
 // NOTE: Should be = the `MAX_RESPONSE_SIZE` defined in `replicated_state/bitcoin.rs`
 // for pagination on the replica side to work as expected.
-const MAX_RESPONSE_SIZE: usize = 2_000_000;
+const MAX_RESPONSE_SIZE: usize = 1_000_000;
 
 // Max number of next block headers that can be returned in the `GetSuccessorsResponse`.
 const MAX_NEXT_BLOCK_HEADERS_LENGTH: usize = 100;
@@ -30,6 +30,12 @@ const MAX_BLOCKS_LENGTH: usize = 100;
 
 const BLOCK_HEADER_SIZE: usize = 80;
 
+// The maximum number of get blocks requests that can be in-flight at any given time.
+// The number of blokcs outside of hte main chain easily exceeds 100 currently.
+// Setting this to 100 would prevent the adapter from making progress,
+// as it gets stuck on these blocks, whose requests timeout indefinitely.
+const MAX_IN_FLIGHT_BLOCKS: usize = 1000;
+
 // The maximum number of bytes the `next` field in a response can take.
 const MAX_NEXT_BYTES: usize = MAX_NEXT_BLOCK_HEADERS_LENGTH * BLOCK_HEADER_SIZE;
 
@@ -96,7 +102,7 @@ impl GetSuccessorsHandler {
             .processed_block_hashes
             .observe(request.processed_block_hashes.len() as f64);
 
-        let response = {
+        let (blocks, next) = {
             let state = self.state.lock().unwrap();
             let anchor_height = state
                 .get_cached_header(&request.anchor)
@@ -115,17 +121,24 @@ impl GetSuccessorsHandler {
                 &request.processed_block_hashes,
                 &blocks,
             );
-            GetSuccessorsResponse { blocks, next }
+            (blocks, next)
+        };
+        //extract the first MAX_NEXT_BLOCK_HEADERS_LENGTH and return them to the caller
+        let response_next = &next[..next.len().min(MAX_NEXT_BLOCK_HEADERS_LENGTH)];
+        let response = GetSuccessorsResponse {
+            blocks,
+            next: response_next.to_vec(),
         };
+
         self.metrics
             .response_blocks
             .observe(response.blocks.len() as f64);
 
-        if !response.next.is_empty() {
+        if !next.is_empty() {
             // TODO: better handling of full channel as the receivers are never closed.
             self.blockchain_manager_tx
                 .try_send(BlockchainManagerRequest::EnqueueNewBlocksToDownload(
-                    response.next.clone(),
+                    next.clone(),
                 ))
                 .ok();
         }
@@ -219,7 +232,7 @@ fn get_next_headers(
 
     let mut next_headers = vec![];
     while let Some(block_hash) = queue.pop_front() {
-        if next_headers.len() >= MAX_NEXT_BLOCK_HEADERS_LENGTH {
+        if next_headers.len() >= MAX_IN_FLIGHT_BLOCKS {
             break;
         }
 
@@ -237,7 +250,7 @@ fn get_next_headers(
 fn are_multiple_blocks_allowed(network: Network, anchor_height: BlockHeight) -> bool {
     match network {
         Network::Bitcoin => anchor_height <= MAINNET_MAX_MULTI_BLOCK_ANCHOR_HEIGHT,
-        Network::Testnet | Network::Signet | Network::Regtest => true,
+        Network::Testnet | Network::Signet | Network::Regtest | Network::Testnet4 => true,
         other => unreachable!("Unsupported network: {:?}", other),
     }
 }

rs/bitcoin/adapter/src/rpc_server.rs

@@ -12,7 +12,7 @@ use ic_btc_service::{
     BtcServiceSendTransactionRequest, BtcServiceSendTransactionResponse,
 };
 use ic_http_endpoints_async_utils::{incoming_from_nth_systemd_socket, incoming_from_path};
-use ic_logger::{debug, ReplicaLogger};
+use ic_logger::{debug, info, ReplicaLogger};
 use ic_metrics::MetricsRegistry;
 use std::convert::{TryFrom, TryInto};
 use std::sync::{Arc, Mutex};
@@ -89,14 +89,14 @@ impl BtcService for BtcServiceImpl {
             .start_timer();
         let _ = self.last_received_tx.send(Some(Instant::now()));
         let inner = request.into_inner();
-        debug!(self.logger, "Received GetSuccessorsRequest: {:?}", inner);
+        info!(self.logger, "Received GetSuccessorsRequest: {:?}", inner);
         let request = inner.try_into()?;
 
         match BtcServiceGetSuccessorsResponse::try_from(
             self.get_successors_handler.get_successors(request).await?,
         ) {
             Ok(res) => {
-                debug!(self.logger, "Sending GetSuccessorsResponse: {:?}", res);
+                info!(self.logger, "Sending GetSuccessorsResponse: {:?}", res);
                 Ok(Response::new(res))
             }
             Err(err) => Err(err),

rs/bitcoin/validation/BUILD.bazel

@@ -14,6 +14,7 @@ DEV_DEPENDENCIES = [
     "@crate_index//:csv",
     "@crate_index//:hex",
     "@crate_index//:proptest",
+    "@crate_index//:rstest",
 ]
 
 MACRO_DEV_DEPENDENCIES = []

rs/bitcoin/validation/Cargo.toml

@@ -15,3 +15,4 @@ hex = { workspace = true }
 [dev-dependencies]
 csv = "1.1"
 proptest = "0.9.4"
+rstest = { workspace = true }

rs/bitcoin/validation/src/constants.rs

@@ -57,11 +57,18 @@ const TESTNET: &[(BlockHeight, &str)] = &[
     (546, "000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70")
 ];
 
+/// Bitcoin testnet checkpoints
+#[rustfmt::skip]
+const TESTNET4: &[(BlockHeight, &str)] = &[
+    (64000, "000000000deb369dca3115f66e208733066f44c8cc177edd4b5f86869e6355b5")
+];
+
 /// Returns the maximum difficulty target depending on the network
 pub fn max_target(network: &Network) -> Target {
     match network {
         Network::Bitcoin => Target::MAX_ATTAINABLE_MAINNET,
         Network::Testnet => Target::MAX_ATTAINABLE_TESTNET,
+        Network::Testnet4 => Target::MAX_ATTAINABLE_TESTNET,
         Network::Regtest => Target::MAX_ATTAINABLE_REGTEST,
         Network::Signet => Target::MAX_ATTAINABLE_SIGNET,
         &other => unreachable!("Unsupported network: {:?}", other),
@@ -72,7 +79,7 @@ pub fn max_target(network: &Network) -> Target {
 /// readjusted in the network after a fixed time interval.
 pub fn no_pow_retargeting(network: &Network) -> bool {
     match network {
-        Network::Bitcoin | Network::Testnet | Network::Signet => false,
+        Network::Bitcoin | Network::Testnet | Network::Signet | Network::Testnet4 => false,
         Network::Regtest => true,
         &other => unreachable!("Unsupported network: {:?}", other),
     }
@@ -83,6 +90,7 @@ pub fn pow_limit_bits(network: &Network) -> CompactTarget {
     CompactTarget::from_consensus(match network {
         Network::Bitcoin => 0x1d00ffff,
         Network::Testnet => 0x1d00ffff,
+        Network::Testnet4 => 0x1d00ffff,
         Network::Regtest => 0x207fffff,
         Network::Signet => 0x1e0377ae,
         &other => unreachable!("Unsupported network: {:?}", other),
@@ -94,7 +102,7 @@ pub fn checkpoints(network: &Network) -> HashMap<BlockHeight, BlockHash> {
     let points = match network {
         Network::Bitcoin => BITCOIN,
         Network::Testnet => TESTNET,
-        Network::Testnet4 => &[],
+        Network::Testnet4 => TESTNET4,
         Network::Signet => &[],
         Network::Regtest => &[],
         _ => &[],
@@ -114,7 +122,7 @@ pub fn latest_checkpoint_height(network: &Network, current_height: BlockHeight)
     let points = match network {
         Network::Bitcoin => BITCOIN,
         Network::Testnet => TESTNET,
-        Network::Testnet4 => &[],
+        Network::Testnet4 => TESTNET4,
         Network::Signet => &[],
         Network::Regtest => &[],
         _ => &[],