test_loop: add a malicious chunk producer test

chain/client/src/chunk_producer.rs

@@ -39,6 +39,10 @@ pub enum AdvProduceChunksMode {
     Valid,
     // Stop producing chunks.
     StopProduce,
+    // Produce chunks but do not include any transactions.
+    ProduceWithoutTx,
+    // Produce chunks but do not bother checking if included transactions pass validity check.
+    ProduceWithoutTxValidityCheck,
 }
 
 pub struct ProduceChunkResult {
@@ -251,13 +255,32 @@ impl ChunkProducer {
                 ))
             })?;
         let last_chunk = self.chain.get_chunk(&last_chunk_header.chunk_hash())?;
-        let prepared_transactions = self.prepare_transactions(
-            shard_uid,
-            prev_block,
-            &last_chunk,
-            chunk_extra.as_ref(),
-            chain_validate,
-        )?;
+        let prepared_transactions = {
+            #[cfg(feature = "test_features")]
+            match self.adv_produce_chunks {
+                Some(AdvProduceChunksMode::ProduceWithoutTx) => PreparedTransactions {
+                    transactions: Vec::new(),
+                    limited_by: None,
+                    storage_proof: None,
+                },
+                _ => self.prepare_transactions(
+                    shard_uid,
+                    prev_block,
+                    &last_chunk,
+                    chunk_extra.as_ref(),
+                    chain_validate,
+                )?,
+            }
+            #[cfg(not(feature = "test_features"))]
+            self.prepare_transactions(
+                shard_uid,
+                prev_block,
+                &last_chunk,
+                chunk_extra.as_ref(),
+                chain_validate,
+            )?
+        };
+
         #[cfg(feature = "test_features")]
         let prepared_transactions = Self::maybe_insert_invalid_transaction(
             prepared_transactions,

chain/client/src/client.rs

@@ -1688,7 +1688,15 @@ impl Client {
                 next_height,
                 shard_id,
                 Some(signer),
-                &self.chain.transaction_validity_check(block.header().clone()),
+                &|tx| {
+                    #[cfg(features = "test_features")]
+                    match self.adv_produce_chunks {
+                        Some(AdvProduceChunksMode::ProduceWithoutTxValidityCheck) => true,
+                        _ => chain.transaction_validity_check(block.header().clone())(tx),
+                    }
+                    #[cfg(not(features = "test_features"))]
+                    self.chain.transaction_validity_check(block.header().clone())(tx)
+                },
             );
             match result {
                 Ok(Some(result)) => {

chain/client/src/client_actor.rs

@@ -407,6 +407,7 @@ pub enum AdvProduceBlockHeightSelection {
 pub enum NetworkAdversarialMessage {
     AdvProduceBlocks(u64, bool),
     AdvProduceChunks(AdvProduceChunksMode),
+    AdvInsertInvalidTransactions(bool),
     AdvSwitchToHeight(u64),
     AdvDisableHeaderSync,
     AdvDisableDoomslug,
@@ -486,6 +487,11 @@ impl Handler<NetworkAdversarialMessage> for ClientActorInner {
                 self.client.chunk_producer.adv_produce_chunks = Some(adv_produce_chunks);
                 None
             }
+            NetworkAdversarialMessage::AdvInsertInvalidTransactions(on) => {
+                info!(target: "adversary", on, "invalid transactions");
+                self.client.produce_invalid_tx_in_chunks = on;
+                None
+            }
         }
     }
 }

core/async/src/test_loop.rs

@@ -453,7 +453,7 @@ impl Drop for TestLoopV2 {
             self.events.clear();
             panic!(
                 "Event scheduled at {} is not handled at the end of the test: {}.
-                 Consider calling `test.shutdown_and_drain_remaining_events(...)`.",
+                     Consider calling `test.shutdown_and_drain_remaining_events(...)`.",
                 event.due, event.event.description
             );
         }

integration-tests/src/test_loop/tests/malicious_chunk_producer.rs

@@ -0,0 +1,142 @@
+#![cfg(feature = "test_features")] // required for adversarial behaviors
+//! Test behaviors of the network when the chunk producer is malicious or misbehaving.
+
+use crate::test_loop::builder::TestLoopBuilder;
+use crate::test_loop::env::TestLoopEnv;
+use crate::test_loop::utils::transactions::get_anchor_hash;
+use crate::test_loop::utils::ONE_NEAR;
+use near_async::messaging::CanSend as _;
+use near_async::time::Duration;
+use near_chain_configs::test_genesis::{
+    build_genesis_and_epoch_config_store, GenesisAndEpochConfigParams, ValidatorsSpec,
+};
+use near_client::client_actor::{AdvProduceChunksMode, NetworkAdversarialMessage};
+use near_client::test_utils::test_loop::ClientQueries;
+use near_client::ProcessTxRequest;
+use near_primitives::shard_layout::ShardLayout;
+use near_primitives::test_utils::create_user_test_signer;
+use near_primitives::transaction::SignedTransaction;
+use near_primitives::types::AccountId;
+use near_primitives::version::PROTOCOL_VERSION;
+use near_primitives::views::{QueryRequest, QueryResponseKind};
+
+#[test]
+fn test_producer_with_expired_transactions() {
+    let builder = TestLoopBuilder::new();
+    let accounts =
+        (0..3).map(|i| format!("account{}", i).parse().unwrap()).collect::<Vec<AccountId>>();
+    let chunk_producer = accounts[0].as_str();
+    let validators: Vec<_> = accounts[1..].iter().map(|a| a.as_str()).collect();
+    let validators_spec = ValidatorsSpec::desired_roles(&[chunk_producer], &validators);
+    let (genesis, epoch_config_store) = build_genesis_and_epoch_config_store(
+        GenesisAndEpochConfigParams {
+            epoch_length: 10,
+            protocol_version: PROTOCOL_VERSION,
+            shard_layout: ShardLayout::simple_v1(&[]),
+            validators_spec,
+            accounts: &accounts,
+        },
+        |genesis_builder| genesis_builder.genesis_height(10000).transaction_validity_period(10),
+        |epoch_config_builder| epoch_config_builder,
+    );
+    let mut test_loop_env = builder
+        .genesis(genesis)
+        .epoch_config_store(epoch_config_store)
+        .clients(accounts.clone())
+        .build();
+    let TestLoopEnv { test_loop, datas: node_datas, .. } = &mut test_loop_env;
+
+    // First we're gonna ask the chunk producer to keep producing empty chunks and send some
+    // transactions as well. This will keep transactions in the transaction pool for more blocks
+    // than the transactions are valid for.
+    let chunk_producer = &node_datas[0];
+    let data_clone = node_datas.clone();
+    test_loop.send_adhoc_event(format!("set chunk production without transactions"), move |_| {
+        data_clone[0].client_sender.send(NetworkAdversarialMessage::AdvProduceChunks(
+            AdvProduceChunksMode::ProduceWithoutTx,
+        ));
+    });
+    for account in &accounts[1..] {
+        let chunk_producer = chunk_producer.clone();
+        let sender = account.clone();
+        let receiver = accounts[0].clone();
+        test_loop.send_adhoc_event(format!("transaction"), move |data| {
+            let signer = create_user_test_signer(&sender);
+            let clients = vec![&data.get(&chunk_producer.client_sender.actor_handle()).client];
+            let response = clients.runtime_query(
+                &receiver,
+                QueryRequest::ViewAccessKey {
+                    account_id: sender.clone(),
+                    public_key: signer.public_key(),
+                },
+            );
+            let QueryResponseKind::AccessKey(access_key) = response.kind else {
+                panic!("Expected AccessKey response");
+            };
+            let anchor_hash = get_anchor_hash(&clients);
+            let tx = SignedTransaction::send_money(
+                access_key.nonce + 1,
+                sender,
+                receiver,
+                &signer,
+                ONE_NEAR,
+                anchor_hash,
+            );
+            let process_tx_request =
+                ProcessTxRequest { transaction: tx, is_forwarded: false, check_only: false };
+            chunk_producer.client_sender.send(process_tx_request);
+        });
+    }
+
+    // Now that we've sent the transactions, wait for 25 chunk productions without transactions
+    // being included.
+    test_loop.run_until(
+        |test_loop_data| {
+            test_loop_data
+                .get(&chunk_producer.client_sender.actor_handle())
+                .client
+                .chain
+                .head()
+                .unwrap()
+                .height
+                > 10025
+        },
+        Duration::seconds(30),
+    );
+
+    // Produce chunks without validity checks! The chunks should contain the transactions, but the
+    // validators should simply discard the transactions.
+    // For a good measure insert some invalid transactions that may be invalid in other ways than
+    // them having been expired.
+    let data_clone = node_datas.clone();
+    test_loop.send_adhoc_event(format!("produce chunks without validity checks"), move |_| {
+        data_clone[0]
+            .client_sender
+            .send(NetworkAdversarialMessage::AdvInsertInvalidTransactions(true));
+        data_clone[0].client_sender.send(NetworkAdversarialMessage::AdvProduceChunks(
+            AdvProduceChunksMode::ProduceWithoutTxValidityCheck,
+        ));
+    });
+    test_loop.run_until(
+        |test_loop_data| {
+            for node in &node_datas[..] {
+                let c = &test_loop_data.get(&node.client_sender.actor_handle()).client;
+                let h = c.chain.head().unwrap().height;
+                if h <= 10050 {
+                    return false;
+                }
+            }
+            true
+        },
+        Duration::seconds(30),
+    );
+
+    // I'd have loved to check this holds true for chunk validators but they do not track shards
+    // and thus cannot provide the info about balances.
+    let clients = vec![&test_loop.data.get(&chunk_producer.client_sender.actor_handle()).client];
+    for account in &accounts {
+        let actual = clients.query_balance(account);
+        assert_eq!(actual, 1000000 * ONE_NEAR, "no transfers should have happened");
+    }
+    test_loop_env.shutdown_and_drain_remaining_events(Duration::seconds(20));
+}

integration-tests/src/test_loop/tests/mod.rs

@@ -11,6 +11,7 @@ mod fix_chunk_producer_stake_threshold;
 mod fix_min_stake_ratio;
 mod fix_stake_threshold;
 mod in_memory_tries;
+mod malicious_chunk_producer;
 mod max_receipt_size;
 mod multinode_stateless_validators;
 mod multinode_test_loop_example;

integration-tests/src/test_loop/utils/transactions.rs

@@ -153,7 +153,7 @@ pub(crate) fn execute_money_transfers(
         .collect_vec();
     for account in accounts {
         let expected = *balances.get(account).unwrap();
-        let actual = clients.query_balance(account);
+        let actual = clients.query_balance(&account);
         if expected != actual {
             return Err(BalanceMismatchError { account: account.clone(), expected, actual });
         }