Blockstore: Migrate ShredIndex type to more efficient data structure (#…

…3900)

ledger/Cargo.toml

@@ -31,6 +31,7 @@ lru = { workspace = true }
 mockall = { workspace = true }
 num_cpus = { workspace = true }
 num_enum = { workspace = true }
+proptest = { workspace = true }
 prost = { workspace = true }
 qualifier_attr = { workspace = true }
 rand = { workspace = true }

ledger/proptest-regressions/blockstore_meta.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc d28b14f167a3950cfc2a5b82dff1e15c65e9ac23a5c249f812e69af96c3489ed # shrinks to coding_indices = 0..0, data_indices = 2984..15152, slot = 0

ledger/src/blockstore_db.rs

@@ -1,8 +1,7 @@
 pub use rocksdb::Direction as IteratorDirection;
 use {
     crate::{
-        blockstore_meta,
-        blockstore_meta::MerkleRootMeta,
+        blockstore_meta::{self, MerkleRootMeta},
         blockstore_metrics::{
             maybe_enable_rocksdb_perf, report_rocksdb_read_perf, report_rocksdb_write_perf,
             BlockstoreRocksDbColumnFamilyMetrics, PerfSamplingStatus, PERF_METRIC_OP_NAME_GET,
@@ -11,7 +10,7 @@ use {
         },
         blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions},
     },
-    bincode::{deserialize, serialize},
+    bincode::{deserialize, Options as BincodeOptions},
     byteorder::{BigEndian, ByteOrder},
     log::*,
     prost::Message,
@@ -796,6 +795,14 @@ pub trait ColumnName {
 
 pub trait TypedColumn: Column {
     type Type: Serialize + DeserializeOwned;
+
+    fn deserialize(data: &[u8]) -> Result<Self::Type> {
+        Ok(bincode::deserialize(data)?)
+    }
+
+    fn serialize(data: &Self::Type) -> Result<Vec<u8>> {
+        Ok(bincode::serialize(data)?)
+    }
 }
 
 impl TypedColumn for columns::AddressSignatures {
@@ -1210,6 +1217,28 @@ impl ColumnName for columns::Index {
 }
 impl TypedColumn for columns::Index {
     type Type = blockstore_meta::Index;
+
+    fn deserialize(data: &[u8]) -> Result<Self::Type> {
+        let config = bincode::DefaultOptions::new()
+            // `bincode::serialize` uses fixint encoding by default, so we need to use the same here
+            .with_fixint_encoding()
+            .reject_trailing_bytes();
+
+        // Migration strategy for new column format:
+        // 1. Release 1: Add ability to read new format as fallback, keep writing old format
+        // 2. Release 2: Switch to writing new format, keep reading old format as fallback
+        // 3. Release 3: Remove old format support once stable
+        // This allows safe downgrade to Release 1 since it can read both formats
+        // https://github.com/anza-xyz/agave/issues/3570
+        let index: bincode::Result<blockstore_meta::Index> = config.deserialize(data);
+        match index {
+            Ok(index) => Ok(index),
+            Err(_) => {
+                let index: blockstore_meta::IndexV2 = config.deserialize(data)?;
+                Ok(index.into())
+            }
+        }
+    }
 }
 
 impl SlotColumn for columns::DeadSlots {}
@@ -1662,7 +1691,7 @@ where
         let result = self
             .backend
             .multi_get_cf(self.handle(), keys)
-            .map(|out| Ok(out?.as_deref().map(deserialize).transpose()?));
+            .map(|out| out?.as_deref().map(C::deserialize).transpose());
 
         if let Some(op_start_instant) = is_perf_enabled {
             // use multi-get instead
@@ -1689,7 +1718,7 @@ where
             &self.read_perf_status,
         );
         if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), key)? {
-            let value = deserialize(pinnable_slice.as_ref())?;
+            let value = C::deserialize(pinnable_slice.as_ref())?;
             result = Ok(Some(value))
         }
 
@@ -1709,7 +1738,7 @@ where
             self.column_options.rocks_perf_sample_interval,
             &self.write_perf_status,
         );
-        let serialized_value = serialize(value)?;
+        let serialized_value = C::serialize(value)?;
 
         let key = Self::key_from_index(index);
         let result = self.backend.put_cf(self.handle(), &key, &serialized_value);
@@ -1732,7 +1761,7 @@ where
         value: &C::Type,
     ) -> Result<()> {
         let key = Self::key_from_index(index);
-        let serialized_value = serialize(value)?;
+        let serialized_value = C::serialize(value)?;
         batch.put_cf(self.handle(), &key, &serialized_value)
     }
 }
@@ -2254,7 +2283,7 @@ pub mod tests {
         C: ColumnIndexDeprecation + TypedColumn + ColumnName,
     {
         pub fn put_deprecated(&self, index: C::DeprecatedIndex, value: &C::Type) -> Result<()> {
-            let serialized_value = serialize(value)?;
+            let serialized_value = C::serialize(value)?;
             self.backend
                 .put_cf(self.handle(), &C::deprecated_key(index), &serialized_value)
         }