v3.1: Raise Entry's VersionedTransaction preallocation limit (backport of anza-xyz#10179) (anza-xyz#10182)

* Raise Entry's VersionedTransaction preallocation limit (anza-xyz#10179)

* Raise Entry's VersionedTransaction preallocation limit

* Add unit tests

* Use MAX_DATA_SHREDS_SIZE prealloc gate in ledger deserialize

(cherry picked from commit b1e2367)

* Update to support wincode 0.1.x

---------

Co-authored-by: Zach Brown <[email protected]>

Author: 2 people

entry/src/entry.rs

@@ -35,7 +35,11 @@ use {
         thread::{self, JoinHandle},
         time::Instant,
     },
-    wincode::{containers::Pod, SchemaRead, SchemaWrite},
+    wincode::{
+        containers::{Elem, Pod, Vec as WincodeVec},
+        len::BincodeLen,
+        SchemaRead, SchemaWrite,
+    },
 };
 
 pub type EntrySender = Sender<Vec<Entry>>;
@@ -91,6 +95,10 @@ pub struct Api<'a> {
         Symbol<'a, unsafe extern "C" fn(hashes: *mut u8, num_hashes: *const u64)>,
 }
 
+const MAX_DATA_SHREDS_PER_SLOT: usize = 32_768;
+pub const MAX_DATA_SHREDS_SIZE: usize = MAX_DATA_SHREDS_PER_SLOT * solana_packet::PACKET_DATA_SIZE;
+pub type MaxDataShredsLen = BincodeLen<MAX_DATA_SHREDS_SIZE>;
+
 /// Each Entry contains three pieces of data. The `num_hashes` field is the number
 /// of hashes performed since the previous entry.  The `hash` field is the result
 /// of hashing `hash` from the previous entry `num_hashes` times.  The `transactions`
@@ -130,7 +138,7 @@ pub struct Entry {
     /// An unordered list of transactions that were observed before the Entry ID was
     /// generated. They may have been observed before a previous Entry ID but were
     /// pushed back into this list to ensure deterministic interpretation of the ledger.
-    #[wincode(with = "Vec<crate::wincode::VersionedTransaction>")]
+    #[wincode(with = "WincodeVec<Elem<crate::wincode::VersionedTransaction>, MaxDataShredsLen>")]
     pub transactions: Vec<VersionedTransaction>,
 }
 

entry/src/wincode.rs

@@ -235,7 +235,7 @@ impl SchemaRead<'_> for VersionedMsg {
 #[cfg(test)]
 mod tests {
     use {
-        crate::entry::Entry,
+        crate::entry::{Entry, MAX_DATA_SHREDS_SIZE},
         proptest::prelude::*,
         solana_address::{Address, ADDRESS_BYTES},
         solana_hash::{Hash, HASH_BYTES},
@@ -441,4 +441,82 @@ mod tests {
             prop_assert_eq!(entries, deserialized);
         }
     }
+
+    #[test]
+    fn entry_deserialize_rejects_excessive_prealloc() {
+        let message = LegacyMessage {
+            header: MessageHeader {
+                num_required_signatures: 0,
+                num_readonly_signed_accounts: 0,
+                num_readonly_unsigned_accounts: 0,
+            },
+            account_keys: vec![],
+            recent_blockhash: Hash::new_from_array([0u8; HASH_BYTES]),
+            instructions: vec![],
+        };
+        let transaction = VersionedTransaction {
+            signatures: vec![],
+            message: VersionedMessage::Legacy(message),
+        };
+        let entry = Entry {
+            num_hashes: 0,
+            hash: Hash::new_from_array([0u8; HASH_BYTES]),
+            transactions: vec![transaction],
+        };
+
+        let mut data = wincode::serialize(&entry).unwrap();
+        let over_limit: usize = MAX_DATA_SHREDS_SIZE / size_of::<VersionedTransaction>() + 1;
+        let len_offset = 8 + HASH_BYTES;
+        // Fudge the length of the vec to be over the limit to trigger the preallocation
+        // size limit error.
+        data[len_offset..len_offset + 8].copy_from_slice(&over_limit.to_le_bytes());
+
+        let needed_bytes = over_limit * size_of::<VersionedTransaction>();
+        let err = Entry::deserialize(&data).unwrap_err();
+        assert!(matches!(
+            err,
+            wincode::error::ReadError::PreallocationSizeLimit {
+                limit: MAX_DATA_SHREDS_SIZE,
+                needed,
+            } if needed == needed_bytes,
+        ));
+    }
+
+    #[test]
+    fn entry_deserialize_accepts_prealloc_at_limit() {
+        let message = LegacyMessage {
+            header: MessageHeader {
+                num_required_signatures: 0,
+                num_readonly_signed_accounts: 0,
+                num_readonly_unsigned_accounts: 0,
+            },
+            account_keys: vec![],
+            recent_blockhash: Hash::new_from_array([0u8; HASH_BYTES]),
+            instructions: vec![],
+        };
+        let transaction = VersionedTransaction {
+            signatures: vec![],
+            message: VersionedMessage::Legacy(message),
+        };
+        let entry = Entry {
+            num_hashes: 0,
+            hash: Hash::new_from_array([0u8; HASH_BYTES]),
+            transactions: vec![transaction],
+        };
+
+        let mut data = wincode::serialize(&entry).unwrap();
+        let at_limit: usize = MAX_DATA_SHREDS_SIZE / size_of::<VersionedTransaction>();
+        let len_offset = 8 + HASH_BYTES;
+        // Fudge the length of the vec to be at the limit.
+        data[len_offset..len_offset + 8].copy_from_slice(&at_limit.to_le_bytes());
+
+        let err = Entry::deserialize(&data).unwrap_err();
+        assert!(!matches!(
+            err,
+            wincode::error::ReadError::PreallocationSizeLimit {
+                limit: _,
+                needed: _,
+            }
+        ));
+    }
 }

ledger/src/blockstore.rs

@@ -37,7 +37,7 @@ use {
     solana_account::ReadableAccount,
     solana_address_lookup_table_interface::state::AddressLookupTable,
     solana_clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND},
-    solana_entry::entry::{create_ticks, Entry},
+    solana_entry::entry::{create_ticks, Entry, MaxDataShredsLen},
     solana_genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE},
     solana_hash::Hash,
     solana_keypair::Keypair,
@@ -82,6 +82,10 @@ use {
     tar,
     tempfile::{Builder, TempDir},
     thiserror::Error,
+    wincode::{
+        containers::{Elem, Vec as WincodeVec},
+        Deserialize as _,
+    },
 };
 
 pub mod blockstore_purge;
@@ -3703,11 +3707,15 @@ impl Blockstore {
                         )))
                     })
                     .and_then(|payload| {
-                        wincode::deserialize::<Vec<Entry>>(&payload).map_err(|e| {
-                            BlockstoreError::InvalidShredData(Box::new(bincode::ErrorKind::Custom(
-                                format!("could not reconstruct entries: {e:?}"),
-                            )))
-                        })
+                        <WincodeVec<Elem<Entry>, MaxDataShredsLen>>::deserialize(&payload).map_err(
+                            |e| {
+                                BlockstoreError::InvalidShredData(Box::new(
+                                    bincode::ErrorKind::Custom(format!(
+                                        "could not reconstruct entries: {e:?}"
+                                    )),
+                                ))
+                            },
+                        )
                     })
             })
             .flatten_ok()