feat(l1): implement flatkeyvalue (snapshots)

cmd/ethrex/Cargo.toml

@@ -26,7 +26,7 @@ ethrex-sdk = { workspace = true, optional = true }
 ethrex-storage.workspace = true
 ethrex-storage-rollup = { workspace = true, optional = true }
 ethrex-vm.workspace = true
-
+ethrex-trie.workspace = true
 tikv-jemallocator = { version = "0.6.0", optional = true, features = [
   "stats",
   "unprefixed_malloc_on_supported_platforms",

crates/common/trie/db.rs

@@ -25,6 +25,9 @@ pub trait TrieDB: Send + Sync {
     fn put(&self, key: Nibbles, value: Vec<u8>) -> Result<(), TrieError> {
         self.put_batch(vec![(key, value)])
     }
+    fn flatkeyvalue_computed(&self, _key: Nibbles) -> bool {
+        false
+    }
 }
 
 /// InMemory implementation for the TrieDB trait, with get and put operations.
@@ -102,15 +105,3 @@ impl TrieDB for InMemoryTrieDB {
         Ok(())
     }
 }
-
-pub fn nibbles_to_fixed_size(nibbles: Nibbles) -> [u8; 33] {
-    let node_hash_ref = nibbles.to_bytes();
-    let original_len = node_hash_ref.len();
-
-    let mut buffer = [0u8; 33];
-
-    // Encode the node as [node_path..., original_len]
-    buffer[32] = nibbles.len() as u8;
-    buffer[..original_len].copy_from_slice(&node_hash_ref);
-    buffer
-}

crates/common/trie/node.rs

@@ -70,6 +70,9 @@ impl NodeRef {
                     Node::Leaf(_) => {}
                 }
                 let hash = *hash.get_or_init(|| node.compute_hash());
+                if let Node::Leaf(leaf) = node.as_ref() {
+                    acc.push((path.concat(&leaf.partial), leaf.value.clone()));
+                }
                 acc.push((path.clone(), node.encode_to_vec()));
 
                 *self = hash.into();

crates/common/trie/trie.rs

@@ -98,6 +98,19 @@ impl Trie {
     pub fn get(&self, pathrlp: &PathRLP) -> Result<Option<ValueRLP>, TrieError> {
         let path = Nibbles::from_bytes(pathrlp);
 
+        if pathrlp.len() == 32
+            && !self.pending_removal.contains(&path)
+            && self.db().flatkeyvalue_computed(path.clone())
+        {
+            let Some(value_rlp) = self.db.get(path)? else {
+                return Ok(None);
+            };
+            if value_rlp.is_empty() {
+                return Ok(None);
+            }
+            return Ok(Some(value_rlp));
+        }
+
         Ok(match self.root {
             NodeRef::Node(ref node, _) => node.get(self.db.as_ref(), path)?,
             NodeRef::Hash(hash) if hash.is_valid() => Node::decode(
@@ -138,6 +151,10 @@ impl Trie {
         if !self.root.is_valid() {
             return Ok(None);
         }
+        if path.len() == 32 {
+            self.pending_removal.insert(Nibbles::from_bytes(path));
+        }
+
         // If the trie is not empty, call the root node's removal logic.
         let (node, value) = self
             .root

crates/common/trie/trie_sorted.rs

@@ -429,6 +429,10 @@ mod test {
         let computed_data = computed_data.lock().unwrap();
         let expected_data = expected_data.lock().unwrap();
         for (k, v) in expected_data.iter() {
+            // skip flatkeyvalues, we don't want them
+            if k.last().cloned() == Some(16) {
+                continue;
+            }
             assert!(computed_data.contains_key(k));
             assert_eq!(*v, computed_data[k]);
         }

crates/networking/p2p/sync.rs

@@ -904,6 +904,8 @@ impl Syncer {
         }
         *METRICS.heal_end_time.lock().await = Some(SystemTime::now());
 
+        store.generate_flatkeyvalue()?;
+
         debug_assert!(validate_state_root(store.clone(), pivot_header.state_root).await);
         debug_assert!(validate_storage_root(store.clone(), pivot_header.state_root).await);
 

crates/storage/api.rs

@@ -375,4 +375,6 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe {
 
     /// Clear all headers downloaded during fullsync
     async fn clear_fullsync_headers(&self) -> Result<(), StoreError>;
+
+    fn generate_flatkeyvalue(&self) -> Result<(), StoreError>;
 }

crates/storage/error.rs

@@ -38,4 +38,6 @@ pub enum StoreError {
     TryInto(#[from] std::num::TryFromIntError),
     #[error("Update batch contains no blocks")]
     UpdateBatchNoBlocks,
+    #[error("Pivot changed")]
+    PivotChanged,
 }

crates/storage/store.rs

@@ -1358,6 +1358,10 @@ impl Store {
     pub async fn clear_fullsync_headers(&self) -> Result<(), StoreError> {
         self.engine.clear_fullsync_headers().await
     }
+
+    pub fn generate_flatkeyvalue(&self) -> Result<(), StoreError> {
+        self.engine.generate_flatkeyvalue()
+    }
 }
 
 pub struct AccountProof {
@@ -1456,17 +1460,19 @@ mod tests {
         F: FnOnce(Store) -> Fut,
         Fut: std::future::Future<Output = ()>,
     {
+        let nonce: u64 = H256::random().to_low_u64_be();
+        let path = format!("store-test-db-{nonce}");
         // Remove preexistent DBs in case of a failed previous test
         if !matches!(engine_type, EngineType::InMemory) {
-            remove_test_dbs("store-test-db");
+            remove_test_dbs(&path);
         };
         // Build a new store
-        let store = Store::new("store-test-db", engine_type).expect("Failed to create test db");
+        let store = Store::new(&path, engine_type).expect("Failed to create test db");
         // Run the test
         test_func(store).await;
         // Remove store (if needed)
         if !matches!(engine_type, EngineType::InMemory) {
-            remove_test_dbs("store-test-db");
+            remove_test_dbs(&path);
         };
     }
 

crates/storage/store_db/in_memory.rs

@@ -728,6 +728,12 @@ impl StoreEngine for Store {
         self.inner()?.fullsync_headers.clear();
         Ok(())
     }
+
+    fn generate_flatkeyvalue(&self) -> Result<(), StoreError> {
+        // FlatKeyValue currently not supported for the InMemory DB
+        // Silently ignoring the request to build the FlatKeyValue is harmless
+        Ok(())
+    }
 }
 
 impl Debug for Store {