Blockstore: Migrate ShredIndex type to more efficient data structure (#…

…3900)

(cherry picked from commit f8e5b16)

# Conflicts:
#	ledger/src/blockstore_db.rs
#	svm/examples/Cargo.lock

ledger/Cargo.toml

@@ -31,6 +31,7 @@ lru = { workspace = true }
 mockall = { workspace = true }
 num_cpus = { workspace = true }
 num_enum = { workspace = true }
+proptest = { workspace = true }
 prost = { workspace = true }
 qualifier_attr = { workspace = true }
 rand = { workspace = true }

ledger/proptest-regressions/blockstore_meta.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc d28b14f167a3950cfc2a5b82dff1e15c65e9ac23a5c249f812e69af96c3489ed # shrinks to coding_indices = 0..0, data_indices = 2984..15152, slot = 0

ledger/src/blockstore_db.rs

@@ -1,8 +1,7 @@
 pub use rocksdb::Direction as IteratorDirection;
 use {
     crate::{
-        blockstore_meta,
-        blockstore_meta::MerkleRootMeta,
+        blockstore_meta::{self, MerkleRootMeta},
         blockstore_metrics::{
             maybe_enable_rocksdb_perf, report_rocksdb_read_perf, report_rocksdb_write_perf,
             BlockstoreRocksDbColumnFamilyMetrics, PerfSamplingStatus, PERF_METRIC_OP_NAME_GET,
@@ -13,7 +12,7 @@ use {
             AccessType, BlockstoreOptions, LedgerColumnOptions, ShredStorageType,
         },
     },
-    bincode::{deserialize, serialize},
+    bincode::{deserialize, Options as BincodeOptions},
     byteorder::{BigEndian, ByteOrder},
     log::*,
     prost::Message,
@@ -790,6 +789,14 @@ pub trait ColumnName {
 
 pub trait TypedColumn: Column {
     type Type: Serialize + DeserializeOwned;
+
+    fn deserialize(data: &[u8]) -> Result<Self::Type> {
+        Ok(bincode::deserialize(data)?)
+    }
+
+    fn serialize(data: &Self::Type) -> Result<Vec<u8>> {
+        Ok(bincode::serialize(data)?)
+    }
 }
 
 impl TypedColumn for columns::AddressSignatures {
@@ -1207,6 +1214,28 @@ impl ColumnName for columns::Index {
 }
 impl TypedColumn for columns::Index {
     type Type = blockstore_meta::Index;
+
+    fn deserialize(data: &[u8]) -> Result<Self::Type> {
+        let config = bincode::DefaultOptions::new()
+            // `bincode::serialize` uses fixint encoding by default, so we need to use the same here
+            .with_fixint_encoding()
+            .reject_trailing_bytes();
+
+        // Migration strategy for new column format:
+        // 1. Release 1: Add ability to read new format as fallback, keep writing old format
+        // 2. Release 2: Switch to writing new format, keep reading old format as fallback
+        // 3. Release 3: Remove old format support once stable
+        // This allows safe downgrade to Release 1 since it can read both formats
+        // https://github.com/anza-xyz/agave/issues/3570
+        let index: bincode::Result<blockstore_meta::Index> = config.deserialize(data);
+        match index {
+            Ok(index) => Ok(index),
+            Err(_) => {
+                let index: blockstore_meta::IndexV2 = config.deserialize(data)?;
+                Ok(index.into())
+            }
+        }
+    }
 }
 
 impl SlotColumn for columns::DeadSlots {}
@@ -1692,11 +1721,31 @@ where
     where
         I: IntoIterator<Item = C::Index>,
     {
+<<<<<<< HEAD
         let keys: Vec<_> = keys.into_iter().map(C::key).collect();
         {
             let is_perf_enabled = maybe_enable_rocksdb_perf(
                 self.column_options.rocks_perf_sample_interval,
                 &self.read_perf_status,
+=======
+        let is_perf_enabled = maybe_enable_rocksdb_perf(
+            self.column_options.rocks_perf_sample_interval,
+            &self.read_perf_status,
+        );
+
+        let result = self
+            .backend
+            .multi_get_cf(self.handle(), keys)
+            .map(|out| out?.as_deref().map(C::deserialize).transpose());
+
+        if let Some(op_start_instant) = is_perf_enabled {
+            // use multi-get instead
+            report_rocksdb_read_perf(
+                C::NAME,
+                PERF_METRIC_OP_NAME_MULTI_GET,
+                &op_start_instant.elapsed(),
+                &self.column_options,
+>>>>>>> f8e5b1672 (Blockstore: Migrate ShredIndex type to more efficient data structure (#3900))
             );
             let result = self
                 .backend
@@ -1728,7 +1777,7 @@ where
             &self.read_perf_status,
         );
         if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), key)? {
-            let value = deserialize(pinnable_slice.as_ref())?;
+            let value = C::deserialize(pinnable_slice.as_ref())?;
             result = Ok(Some(value))
         }
 
@@ -1748,7 +1797,7 @@ where
             self.column_options.rocks_perf_sample_interval,
             &self.write_perf_status,
         );
-        let serialized_value = serialize(value)?;
+        let serialized_value = C::serialize(value)?;
 
         let result = self
             .backend
@@ -1764,6 +1813,20 @@ where
         }
         result
     }
+<<<<<<< HEAD
+=======
+
+    pub fn put_in_batch(
+        &self,
+        batch: &mut WriteBatch,
+        index: C::Index,
+        value: &C::Type,
+    ) -> Result<()> {
+        let key = Self::key_from_index(index);
+        let serialized_value = C::serialize(value)?;
+        batch.put_cf(self.handle(), &key, &serialized_value)
+    }
+>>>>>>> f8e5b1672 (Blockstore: Migrate ShredIndex type to more efficient data structure (#3900))
 }
 
 impl<C> LedgerColumn<C>
@@ -2370,8 +2433,13 @@ pub mod tests {
     where
         C: ColumnIndexDeprecation + TypedColumn + ColumnName,
     {
+<<<<<<< HEAD
         pub fn put_deprecated(&self, key: C::DeprecatedIndex, value: &C::Type) -> Result<()> {
             let serialized_value = serialize(value)?;
+=======
+        pub fn put_deprecated(&self, index: C::DeprecatedIndex, value: &C::Type) -> Result<()> {
+            let serialized_value = C::serialize(value)?;
+>>>>>>> f8e5b1672 (Blockstore: Migrate ShredIndex type to more efficient data structure (#3900))
             self.backend
                 .put_cf(self.handle(), &C::deprecated_key(key), &serialized_value)
         }