Implement update_chain_tip and document the scan process.

zcash_client_backend/CHANGELOG.md

@@ -16,7 +16,7 @@ and this library adheres to Rust's notion of
   - `ScannedBlock`
   - `ShieldedProtocol`
   - `WalletCommitmentTrees`
-  - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}`
+  - `WalletRead::{chain_tip, block_metadata, block_fully_scanned, suggest_scan_ranges}`
   - `WalletWrite::put_blocks`
   - `chain::CommitmentTreeRoot`
   - `testing::MockWalletDb::new`
@@ -36,8 +36,10 @@ and this library adheres to Rust's notion of
     and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`.
   - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32`
   - `chain::scan_cached_blocks` now takes a `from_height` argument that
-    permits the caller to control the starting position of the scan range
+    permits the caller to control the starting position of the scan range.
     In addition, the `limit` parameter is now required.
+  - `chain::BlockSource::with_blocks` now takes its limit as an `Option<usize>`
+    instead of `Option<u32>`.
   - A new `CommitmentTree` variant has been added to `data_api::error::Error`
   - `data_api::wallet::{create_spend_to_address, create_proposed_transaction,
     shield_transparent_funds}` all now require that `WalletCommitmentTrees` be
@@ -81,8 +83,6 @@ and this library adheres to Rust's notion of
     feature flag, has been modified by the addition of a `sapling_tree` property. 
   - `wallet::input_selection`:
     - `Proposal::target_height` (use `Proposal::min_target_height` instead).
-- `zcash_client_backend::data_api::chain::validate_chain` TODO: document how
-  to handle validation given out-of-order blocks.
 - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been 
   replaced by `zcash_client_backend::scanning::ScanError`
 - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}`

zcash_client_backend/proto/service.proto

@@ -118,6 +118,22 @@ message TreeState {
     string orchardTree = 6; // orchard commitment tree state
 }
 
+enum ShieldedProtocol {
+    sapling = 0;
+    orchard = 1;
+}
+
+message GetSubtreeRootsArg {
+    uint32 startIndex = 1;                  // Index identifying where to start returning subtree roots
+    ShieldedProtocol shieldedProtocol = 2;  // Shielded protocol to return subtree roots for
+    uint32 maxEntries = 3;                  // Maximum number of entries to return, or 0 for all entries.
+}
+message SubtreeRoot {
+    bytes rootHash = 2;                     // The 32-byte Merkle root of the subtree.
+    bytes completingBlockHash = 3;          // The hash of the block that completed this subtree.
+    uint64 completingBlockHeight = 4;       // The height of the block that completed this subtree in the main chain.
+}
+
 // Results are sorted by height, which makes it easy to issue another
 // request that picks up from where the previous left off.
 message GetAddressUtxosArg {
@@ -142,8 +158,12 @@ service CompactTxStreamer {
     rpc GetLatestBlock(ChainSpec) returns (BlockID) {}
     // Return the compact block corresponding to the given block identifier
     rpc GetBlock(BlockID) returns (CompactBlock) {}
+    // Same as GetBlock except actions contain only nullifiers
+    rpc GetBlockNullifiers(BlockID) returns (CompactBlock) {}
     // Return a list of consecutive compact blocks
     rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {}
+    // Same as GetBlockRange except actions contain only nullifiers
+    rpc GetBlockRangeNullifiers(BlockRange) returns (stream CompactBlock) {}
 
     // Return the requested full (not compact) transaction (as from zcashd)
     rpc GetTransaction(TxFilter) returns (RawTransaction) {}
@@ -177,6 +197,10 @@ service CompactTxStreamer {
     rpc GetTreeState(BlockID) returns (TreeState) {}
     rpc GetLatestTreeState(Empty) returns (TreeState) {}
 
+    // Returns a stream of information about roots of subtrees of the Sapling and Orchard
+    // note commitment trees.
+    rpc GetSubtreeRoots(GetSubtreeRootsArg) returns (stream SubtreeRoot) {}
+
     rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {}
     rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {}
 

zcash_client_backend/src/data_api.rs

@@ -502,10 +502,11 @@ pub trait WalletWrite: WalletRead {
     /// Updates the wallet's view of the blockchain.
     ///
     /// This method is used to provide the wallet with information about the state of the
-    /// blockchain. It should be called on wallet startup prior to calling
+    /// blockchain, and detect any previously scanned that needs to be re-validated before
+    /// proceeding with scanning. It should be called at wallet startup prior to calling
     /// [`WalletRead::suggest_scan_ranges`] in order to provide the wallet with the information it
     /// needs to correctly prioritize scanning operations.
-    fn update_chain_tip(&mut self, block_metadata: BlockMetadata) -> Result<(), Self::Error>;
+    fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error>;
 
     /// Caches a decrypted transaction in the persistent wallet store.
     fn store_decrypted_tx(
@@ -782,7 +783,7 @@ pub mod testing {
             Ok(vec![])
         }
 
-        fn update_chain_tip(&mut self, _block_metadata: BlockMetadata) -> Result<(), Self::Error> {
+        fn update_chain_tip(&mut self, _tip_height: BlockHeight) -> Result<(), Self::Error> {
             Ok(())
         }
 

zcash_client_backend/src/data_api/chain.rs

@@ -7,18 +7,21 @@
 //! # #[cfg(feature = "test-dependencies")]
 //! # {
 //! use zcash_primitives::{
-//!     consensus::{BlockHeight, Network, Parameters}
+//!     consensus::{BlockHeight, Network, Parameters},
+//!     sapling
 //! };
 //!
 //! use zcash_client_backend::{
 //!     data_api::{
 //!         WalletRead, WalletWrite,
 //!         chain::{
 //!             BlockSource,
+//!             CommitmentTreeRoot,
 //!             error::Error,
 //!             scan_cached_blocks,
 //!             testing as chain_testing,
 //!         },
+//!         scanning::ScanPriority,
 //!         testing,
 //!     },
 //! };
@@ -32,20 +35,105 @@
 //! # fn test() -> Result<(), Error<(), Infallible>> {
 //! let network = Network::TestNetwork;
 //! let block_source = chain_testing::MockBlockSource;
-//! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork);
+//! let mut wallet_db = testing::MockWalletDb::new(Network::TestNetwork);
 //!
-//! // 1) Download new CompactBlocks into block_source.
-//! //
-//! // 2) FIXME: Obtain necessary block metadata for continuity checking?
-//! //
-//! // 3) Scan cached blocks.
-//! //
-//! // FIXME: update documentation on how to detect when a rewind is required.
-//! //
-//! // At this point, the cache and scanned data are locally consistent (though not
-//! // necessarily consistent with the latest chain tip - this would be discovered the
-//! // next time this codepath is executed after new blocks are received).
-//! scan_cached_blocks(&network, &block_source, &mut db_data, BlockHeight::from(0), 10)
+//! // 1) Download note commitment tree data from lightwalletd
+//! let roots: Vec<CommitmentTreeRoot<sapling::Node>> = unimplemented!();
+//!
+//! // 2) Download chain tip metadata from lightwalletd
+//! let tip_height: BlockHeight = unimplemented!();
+//!
+//! // 3) Notify the wallet of the updated chain tip.
+//! wallet_db.update_chain_tip(tip_height).map_err(Error::Wallet)?;
+//!
+//! // 4) Get the suggested scan ranges from the wallet database
+//! let mut scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?;
+//!
+//! // 5) Run the following loop until the wallet's view of the chain tip as of the previous wallet
+//! //    session is valid.
+//! loop {
+//!     // If there is a range of blocks that needs to be verified, it will always be returned as
+//!     // the first element of the vector of suggested ranges.
+//!     match scan_ranges.first() {
+//!         Some(scan_range) if scan_range.priority() == ScanPriority::Verify => {
+//!             // Download the blocks in `scan_range` into the block source
+//!             unimplemented!();
+//!
+//!             // Scan the downloaded blocks
+//!             let scan_result = scan_cached_blocks(
+//!                 &network,
+//!                 &block_source,
+//!                 &mut wallet_db,
+//!                 scan_range.block_range().start,
+//!                 scan_range.len()
+//!             );
+//!
+//!             // Check for scanning errors that indicate that the wallet's chain tip is out of
+//!             // sync with blockchain history.
+//!             match scan_result {
+//!                 Ok(_) => {
+//!                     // At this point, the cache and scanned data are locally consistent (though
+//!                     // not necessarily consistent with the latest chain tip - this would be
+//!                     // discovered the next time this codepath is executed after new blocks are
+//!                     // received) so we can break out of the loop.
+//!                     break;
+//!                 }
+//!                 Err(Error::Scan(err)) if err.is_continuity_error() => {
+//!                     // Pick a height to rewind to, which must be at least one block before
+//!                     // the height at which the error occurred, but may be an earlier height
+//!                     // determined based on heuristics such as the platform, available bandwidth,
+//!                     // size of recent CompactBlocks, etc.
+//!                     let rewind_height = err.at_height().saturating_sub(10);
+//!
+//!                     // Rewind to the chosen height.
+//!                     wallet_db.truncate_to_height(rewind_height).map_err(Error::Wallet)?;
+//!
+//!                     // Delete cached blocks from rewind_height onwards.
+//!                     //
+//!                     // This does imply that assumed-valid blocks will be re-downloaded, but it
+//!                     // is also possible that in the intervening time, a chain reorg has
+//!                     // occurred that orphaned some of those blocks.
+//!                     unimplemented!();
+//!                 }
+//!                 Err(other) => {
+//!                     // Handle or return other errors
+//!                 }
+//!             }
+//!
+//!             // Truncation will have updated the suggested scan ranges, so we now
+//!             // re_request
+//!             scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?;
+//!         }
+//!         _ => {
+//!             // Nothing to verify; break out of the loop
+//!             break;
+//!         }
+//!     }
+//! }
+//!
+//! // 5) Loop over the remaining suggested scan ranges, retrieving the requested data and calling
+//! //    `scan_cached_blocks` on each range. Periodically, or if a continuity error is
+//! //    encountered, this process should be repeated starting at step (2).
+//! let scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?;
+//! for scan_range in scan_ranges {
+//!     // Download the blocks in `scan_range` into the block source. While in this example this
+//!     // step is performed in-line, it's fine for the download of scan ranges to be asynchronous
+//!     // and for the scanner to process the downloaded ranges as they become available in a
+//!     // separate thread.
+//!     unimplemented!();
+//!
+//!     // Scan the downloaded blocks,
+//!     let scan_result = scan_cached_blocks(
+//!         &network,
+//!         &block_source,
+//!         &mut wallet_db,
+//!         scan_range.block_range().start,
+//!         scan_range.len()
+//!     )?;
+//!
+//!     // Handle scan errors, etc.
+//! }
+//! # Ok(())
 //! # }
 //! # }
 //! ```
@@ -58,14 +146,12 @@ use zcash_primitives::{
 };
 
 use crate::{
-    data_api::{NullifierQuery, WalletWrite},
+    data_api::{BlockMetadata, NullifierQuery, WalletWrite},
     proto::compact_formats::CompactBlock,
     scan::BatchRunner,
     scanning::{add_block_to_runner, check_continuity, scan_block_with_runner},
 };
 
-use super::BlockMetadata;
-
 pub mod error;
 use error::Error;
 
@@ -114,7 +200,7 @@ pub trait BlockSource {
     fn with_blocks<F, WalletErrT>(
         &self,
         from_height: Option<BlockHeight>,
-        limit: Option<u32>,
+        limit: Option<usize>,
         with_row: F,
     ) -> Result<(), error::Error<WalletErrT, Self::Error>>
     where
@@ -145,7 +231,7 @@ pub fn scan_cached_blocks<ParamsT, DbT, BlockSourceT>(
     block_source: &BlockSourceT,
     data_db: &mut DbT,
     from_height: BlockHeight,
-    limit: u32,
+    limit: usize,
 ) -> Result<(), Error<DbT::Error, BlockSourceT::Error>>
 where
     ParamsT: consensus::Parameters + Send + 'static,
@@ -292,7 +378,7 @@ pub mod testing {
         fn with_blocks<F, DbErrT>(
             &self,
             _from_height: Option<BlockHeight>,
-            _limit: Option<u32>,
+            _limit: Option<usize>,
             _with_row: F,
         ) -> Result<(), Error<DbErrT, Infallible>>
         where

zcash_client_backend/src/data_api/scanning.rs

@@ -26,6 +26,7 @@ pub struct ScanRange {
 
 impl ScanRange {
     pub fn from_parts(block_range: Range<BlockHeight>, priority: ScanPriority) -> Self {
+        assert!(block_range.end >= block_range.start);
         ScanRange {
             block_range,
             priority,
@@ -38,6 +39,10 @@ impl ScanRange {
     pub fn priority(&self) -> ScanPriority {
         self.priority
     }
+    pub fn len(&self) -> usize {
+        usize::try_from(u32::from(self.block_range.end) - u32::from(self.block_range.start))
+            .unwrap()
+    }
 
     pub fn truncate_left(&self, block_height: BlockHeight) -> Option<Self> {
         if block_height >= self.block_range.end {