fix(storage): fix max object size

config/default_config.json

@@ -277,7 +277,7 @@
   "storage.mmap_file_config.max_object_size": {
     "description": "The maximum size of a single object in the file in bytes",
     "privacy": "Public",
-    "value": 1048576
+    "value": 268435456
   },
   "storage.mmap_file_config.max_size": {
     "description": "The maximum size of a memory mapped file in bytes. Must be greater than growth_step.",

crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap

@@ -320,7 +320,7 @@ expression: dumped_default_config
   "storage.mmap_file_config.max_object_size": {
     "description": "The maximum size of a single object in the file in bytes",
     "value": {
-      "$serde_json::private::Number": "1048576"
+      "$serde_json::private::Number": "268435456"
     },
     "privacy": "Public"
   },

crates/papyrus_storage/src/compression_utils.rs

@@ -9,7 +9,9 @@ use crate::db::serialization::{StorageSerde, StorageSerdeError};
 // TODO: fine tune the compression hyperparameters (and maybe even the compression algorithm).
 
 // The maximum size of the decompressed data.
-const MAX_DECOMPRESSED_SIZE: usize = 1 << 20; // 1MB
+// TODO(Dvir): consider defining this for each type separately and pass it as an argument to the
+// decompress function.
+pub(crate) const MAX_DECOMPRESSED_SIZE: usize = 1 << 20; // 256 MB
 // The compression level to use. Higher levels are slower but compress better.
 const COMPRESSION_LEVEL: i32 = zstd::DEFAULT_COMPRESSION_LEVEL;
 

crates/papyrus_storage/src/db/serialization.rs

@@ -33,6 +33,8 @@ impl<T: StorageSerde> StorageSerdeEx for T {
 pub trait StorageSerde: Sized {
     fn serialize_into(&self, res: &mut impl std::io::Write) -> Result<(), StorageSerdeError>;
 
+    // TODO(dan): consider returning a result here. Most of the time we probably transform this into
+    // DbError::InnerDeserialization, so having more context might be useful.
     fn deserialize_from(bytes: &mut impl std::io::Read) -> Option<Self>;
 }
 

crates/papyrus_storage/src/mmap_file/mod.rs

@@ -75,7 +75,7 @@ impl Default for MmapFileConfig {
         Self {
             max_size: 1 << 40,        // 1TB
             growth_step: 1 << 30,     // 1GB
-            max_object_size: 1 << 20, // 1MB
+            max_object_size: 1 << 28, // 256MB
         }
     }
 }

crates/papyrus_storage/src/serialization/serializers.rs

@@ -123,6 +123,7 @@ use starknet_api::transaction::{
     TransactionVersion,
 };
 use starknet_types_core::felt::Felt;
+use tracing::warn;
 
 use crate::body::events::EventIndex;
 use crate::body::TransactionIndex;
@@ -965,6 +966,13 @@ impl StorageSerde for DeprecatedContractClass {
         let mut to_compress: Vec<u8> = Vec::new();
         self.abi.serialize_into(&mut to_compress)?;
         self.program.serialize_into(&mut to_compress)?;
+        if to_compress.len() > crate::compression_utils::MAX_DECOMPRESSED_SIZE {
+            warn!(
+                "DeprecatedContractClass serialization size is too large and will lead to \
+                 deserialization error: {}",
+                to_compress.len()
+            );
+        }
         let compressed = compress(to_compress.as_slice())?;
         compressed.serialize_into(res)?;
         self.entry_points_by_type.serialize_into(res)?;
@@ -973,7 +981,8 @@ impl StorageSerde for DeprecatedContractClass {
 
     fn deserialize_from(bytes: &mut impl std::io::Read) -> Option<Self> {
         let compressed_data = Vec::<u8>::deserialize_from(bytes)?;
-        let data = decompress(compressed_data.as_slice()).ok()?;
+        let data = decompress(compressed_data.as_slice())
+            .expect("destination buffer should be large enough");
         let data = &mut data.as_slice();
         Some(Self {
             abi: Option::<Vec<ContractClassAbiEntry>>::deserialize_from(data)?,
@@ -1018,7 +1027,13 @@ impl StorageSerde for CasmContractClass {
         self.hints.serialize_into(&mut to_compress)?;
         self.pythonic_hints.serialize_into(&mut to_compress)?;
         self.entry_points_by_type.serialize_into(&mut to_compress)?;
-
+        if to_compress.len() > crate::compression_utils::MAX_DECOMPRESSED_SIZE {
+            warn!(
+                "CasmContractClass serialization size is too large and will lead to \
+                 deserialization error: {}",
+                to_compress.len()
+            );
+        }
         let compressed = compress(to_compress.as_slice())?;
         compressed.serialize_into(res)?;
 
@@ -1027,7 +1042,8 @@ impl StorageSerde for CasmContractClass {
 
     fn deserialize_from(bytes: &mut impl std::io::Read) -> Option<Self> {
         let compressed_data = Vec::<u8>::deserialize_from(bytes)?;
-        let data = decompress(compressed_data.as_slice()).ok()?;
+        let data = decompress(compressed_data.as_slice())
+            .expect("destination buffer should be large enough");
         let data = &mut data.as_slice();
         Some(Self {
             prime: BigUint::deserialize_from(data)?,
@@ -1053,14 +1069,22 @@ impl StorageSerde for ThinStateDiff {
         self.deprecated_declared_classes.serialize_into(&mut to_compress)?;
         self.nonces.serialize_into(&mut to_compress)?;
         self.replaced_classes.serialize_into(&mut to_compress)?;
+        if to_compress.len() > crate::compression_utils::MAX_DECOMPRESSED_SIZE {
+            warn!(
+                "ThinStateDiff serialization size is too large and will lead to deserialization \
+                 error: {}",
+                to_compress.len()
+            );
+        }
         let compressed = compress(to_compress.as_slice())?;
         compressed.serialize_into(res)?;
         Ok(())
     }
 
     fn deserialize_from(bytes: &mut impl std::io::Read) -> Option<Self> {
         let compressed_data = Vec::<u8>::deserialize_from(bytes)?;
-        let data = decompress(compressed_data.as_slice()).ok()?;
+        let data = decompress(compressed_data.as_slice())
+            .expect("destination buffer should be large enough");
         let data = &mut data.as_slice();
         Some(Self {
             deployed_contracts: IndexMap::deserialize_from(data)?,
@@ -1088,6 +1112,14 @@ macro_rules! auto_storage_serde_conditionally_compressed {
                 )*
                 if to_compress.len() > COMPRESSION_THRESHOLD_BYTES {
                     IsCompressed::Yes.serialize_into(res)?;
+                    if to_compress.len() > crate::compression_utils::MAX_DECOMPRESSED_SIZE {
+                        warn!(
+                            "{} serialization size is too large and will lead to deserialization \
+                            error: {}",
+                            stringify!($name),
+                            to_compress.len()
+                        );
+                    }
                     let compressed = compress(to_compress.as_slice())?;
                     compressed.serialize_into(res)?;
                 } else {
@@ -1101,7 +1133,9 @@ macro_rules! auto_storage_serde_conditionally_compressed {
                 let maybe_compressed_data = Vec::<u8>::deserialize_from(bytes)?;
                 let data = match is_compressed {
                     IsCompressed::No => maybe_compressed_data,
-                    IsCompressed::Yes => decompress(maybe_compressed_data.as_slice()).ok()?,
+                    IsCompressed::Yes => decompress(
+                        maybe_compressed_data.as_slice())
+                            .expect("destination buffer should be large enough"),
                 };
                 let data = &mut data.as_slice();
                 Some(Self {