diff --git a/client/db/src/kv/mod.rs b/client/db/src/kv/mod.rs index 6b72b81b4c..b31be3e3b7 100644 --- a/client/db/src/kv/mod.rs +++ b/client/db/src/kv/mod.rs @@ -23,7 +23,10 @@ mod utils; use std::{ marker::PhantomData, path::{Path, PathBuf}, - sync::Arc, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; use parking_lot::Mutex; @@ -43,7 +46,20 @@ const DB_HASH_LEN: usize = 32; pub type DbHash = [u8; DB_HASH_LEN]; /// Maximum number of blocks inspected in a single recovery pass when the /// latest indexed canonical pointer is stale or missing. +#[cfg(not(test))] const INDEXED_RECOVERY_SCAN_LIMIT: u64 = 8192; +/// Smaller test-only limit so deep-lag branch behavior can be exercised +/// without creating thousands of blocks in unit tests. +#[cfg(test)] +const INDEXED_RECOVERY_SCAN_LIMIT: u64 = 8; +/// Scan limit for the deep-recovery pass when pointer and 32k scan both miss but +/// best > 0. Extends coverage to ~64k blocks from best before falling back to genesis. +const INDEXED_DEEP_RECOVERY_SCAN_LIMIT: u64 = INDEXED_RECOVERY_SCAN_LIMIT * 8; +/// Minimum interval (seconds) between exhaustive-fallback warnings to avoid +/// flooding logs during startup or sustained indexing lag. +const EXHAUSTIVE_FALLBACK_WARN_INTERVAL_SECS: u64 = 60; +/// Epoch-seconds of the last exhaustive-fallback warning. Zero means "never warned". +static LAST_EXHAUSTIVE_WARN_SECS: AtomicU64 = AtomicU64::new(0); /// Database settings. pub struct DatabaseSettings { @@ -131,20 +147,92 @@ impl> fc_api::Backend for Backend< return Ok(canonical_hash); } - // Use cached pointer only as a lower bound hint for recovery scan. - let min_block_hint = self - .mapping - .latest_canonical_indexed_block_number()? - .map(|cached| cached.min(best_number)); + // Walk backwards from best in three layers, each covering a deeper + // non-overlapping range. The persisted pointer is checked last so it + // never short-circuits past higher indexed blocks in the scan ranges. + let bounded_start = best_number.saturating_sub(1); - // Best block is not indexed yet or mapping is stale (reorg). Walk back to - // the latest indexed canonical block and persist the recovered pointer. - if let Some((recovered_number, recovered_hash)) = - self.find_latest_indexed_canonical_block(best_number.saturating_sub(1), min_block_hint)? + // Layer 1 — bounded scan: [best-1 .. best-8k] + if let Some((found_number, found_hash)) = + self.find_latest_indexed_canonical_block(bounded_start, INDEXED_RECOVERY_SCAN_LIMIT)? { self.mapping - .set_latest_canonical_indexed_block(recovered_number)?; - return Ok(recovered_hash); + .set_latest_canonical_indexed_block(found_number)?; + return Ok(found_hash); + } + + // Layer 2 — exhaustive scan: [best-8k-1 .. best-32k] + // Extends the search when indexing is far behind. Skip only at genesis. + if best_number > 0 { + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let prev = LAST_EXHAUSTIVE_WARN_SECS.load(Ordering::Relaxed); + if now_secs.saturating_sub(prev) >= EXHAUSTIVE_FALLBACK_WARN_INTERVAL_SECS + && LAST_EXHAUSTIVE_WARN_SECS + .compare_exchange(prev, now_secs, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + log::warn!( + target: "frontier-db", + "latest_block_hash: exhaustive fallback triggered (best_number={best_number}). If this persists, check indexing progress.", + ); + } + + // 8k bounded + 24k exhaustive = 32k total non-overlapping coverage. + let exhaustive_start = bounded_start.saturating_sub(INDEXED_RECOVERY_SCAN_LIMIT); + let exhaustive_limit = INDEXED_RECOVERY_SCAN_LIMIT * 3; + if let Some((found_number, found_hash)) = + self.find_latest_indexed_canonical_block(exhaustive_start, exhaustive_limit)? + { + self.mapping + .set_latest_canonical_indexed_block(found_number)?; + return Ok(found_hash); + } + } + + // Layer 3 — deep recovery: when best > 0, scan further back before pointer/genesis. + // Covers [best-96k .. best-32k-1]. Must run before pointer so a stale-but-valid pointer + // does not mask a newer indexed block in the deep range. + if best_number > 0 { + let deep_start = bounded_start + .saturating_sub(INDEXED_RECOVERY_SCAN_LIMIT) + .saturating_sub(INDEXED_RECOVERY_SCAN_LIMIT * 3); + if let Some((found_number, found_hash)) = self + .find_latest_indexed_canonical_block(deep_start, INDEXED_DEEP_RECOVERY_SCAN_LIMIT)? + { + self.mapping + .set_latest_canonical_indexed_block(found_number)?; + return Ok(found_hash); + } + } + + // Layer 4 — persisted pointer: O(1) jump when all scans miss (indexing >96k behind). + // Checked after deep recovery so we never return an older block when a newer one + // exists in the deep window. + // + // When the pointer target is stale (e.g. reorg), walk backward from it to + // find the latest valid indexed canonical block instead of falling to genesis. + if let Some(persisted_number) = self.mapping.latest_canonical_indexed_block_number()? { + if persisted_number <= best_number { + if let Some(canonical_hash) = self.indexed_canonical_hash_at(persisted_number)? { + return Ok(canonical_hash); + } + // Pointer target is stale; backtrack from pointer-1 to find a valid block. + if persisted_number > 0 { + let backtrack_start = persisted_number.saturating_sub(1); + if let Some((found_number, found_hash)) = self + .find_latest_indexed_canonical_block( + backtrack_start, + INDEXED_RECOVERY_SCAN_LIMIT, + )? { + self.mapping + .set_latest_canonical_indexed_block(found_number)?; + return Ok(found_hash); + } + } + } } Ok(self.client.info().genesis_hash) @@ -261,18 +349,16 @@ impl> Backend { } /// Finds the latest indexed block that is on the canonical chain by walking - /// backwards from `start_block`, bounded to `INDEXED_RECOVERY_SCAN_LIMIT` - /// probes to keep lookups fast on long chains. + /// backwards from `start_block`, bounded to `scan_limit` probes. fn find_latest_indexed_canonical_block( &self, start_block: u64, - min_block_hint: Option, + scan_limit: u64, ) -> Result, String> { - let scan_limit = INDEXED_RECOVERY_SCAN_LIMIT.saturating_sub(1); - let min_block = start_block - .saturating_sub(scan_limit) - .max(min_block_hint.unwrap_or(0)) - .min(start_block); + if scan_limit == 0 { + return Ok(None); + } + let min_block = start_block.saturating_sub(scan_limit - 1); for block_number in (min_block..=start_block).rev() { if let Some(canonical_hash) = self.indexed_canonical_hash_at(block_number)? { return Ok(Some((block_number, canonical_hash))); @@ -560,3 +646,383 @@ impl MappingDb { self.db.commit(transaction).map_err(|e| e.to_string()) } } + +#[cfg(test)] +mod tests { + use super::*; + + use fc_api::Backend as _; + use sc_block_builder::BlockBuilderBuilder; + use sp_consensus::BlockOrigin; + use sp_core::H256; + use sp_runtime::{generic::Header, traits::BlakeTwo256, Digest}; + use substrate_test_runtime_client::{ + ClientBlockImportExt, DefaultTestClientBuilderExt, TestClientBuilder, + }; + use tempfile::tempdir; + + type OpaqueBlock = sp_runtime::generic::Block< + Header, + substrate_test_runtime_client::runtime::Extrinsic, + >; + + struct TestEnv { + client: Arc, + backend: Arc>, + substrate_hashes: Vec<::Hash>, + _tmp: tempfile::TempDir, + } + + impl TestEnv { + async fn new(num_blocks: u64) -> Self { + let tmp = tempdir().expect("create a temporary directory"); + let (client, _substrate_backend) = TestClientBuilder::new() + .build_with_native_executor::( + None, + ); + let client = Arc::new(client); + + let backend = Arc::new( + Backend::::new( + client.clone(), + &DatabaseSettings { + #[cfg(feature = "rocksdb")] + source: sc_client_db::DatabaseSource::RocksDb { + path: tmp.path().to_path_buf(), + cache_size: 0, + }, + #[cfg(not(feature = "rocksdb"))] + source: sc_client_db::DatabaseSource::ParityDb { + path: tmp.path().to_path_buf(), + }, + }, + ) + .expect("frontier backend"), + ); + + let mut substrate_hashes = vec![client.chain_info().genesis_hash]; + for _ in 1u64..=num_blocks { + let chain_info = client.chain_info(); + let block = BlockBuilderBuilder::new(&*client) + .on_parent_block(chain_info.best_hash) + .with_parent_block_number(chain_info.best_number) + .with_inherent_digests(Digest::default()) + .build() + .unwrap() + .build() + .unwrap() + .block; + let hash = block.header.hash(); + client.import(BlockOrigin::Own, block).await.unwrap(); + substrate_hashes.push(hash); + } + + Self { + client, + backend, + substrate_hashes, + _tmp: tmp, + } + } + + fn index_block(&self, n: u64) { + let eth_hash = H256::repeat_byte(n as u8); + let commitment = MappingCommitment:: { + block_hash: self.substrate_hashes[n as usize], + ethereum_block_hash: eth_hash, + ethereum_transaction_hashes: vec![], + }; + self.backend + .mapping() + .write_hashes(commitment, n, NumberMappingWrite::Write) + .expect("write mapping"); + } + + fn write_stale_mapping(&self, n: u64) { + let stale_eth_hash = H256::repeat_byte(0xA0 + n as u8); + self.backend + .mapping() + .set_block_hash_by_number(n, stale_eth_hash) + .expect("write stale number mapping"); + } + + fn set_pointer(&self, n: u64) { + self.backend + .mapping() + .set_latest_canonical_indexed_block(n) + .expect("set pointer"); + } + + fn genesis_hash(&self) -> ::Hash { + self.client.chain_info().genesis_hash + } + + async fn latest(&self) -> ::Hash { + self.backend + .latest_block_hash() + .await + .expect("latest_block_hash") + } + } + + #[tokio::test] + async fn fast_path_returns_best_when_fully_indexed() { + let env = TestEnv::new(5).await; + for n in 1u64..=5 { + env.index_block(n); + } + env.set_pointer(5); + + let result = env.latest().await; + assert_eq!(result, env.substrate_hashes[5]); + } + + #[tokio::test] + async fn bounded_scan_finds_latest_indexed_under_normal_lag() { + let env = TestEnv::new(10).await; + for n in 1u64..=7 { + env.index_block(n); + } + env.set_pointer(7); + + let result = env.latest().await; + assert_eq!( + result, env.substrate_hashes[7], + "should find block 7 via bounded scan even though best is 10" + ); + } + + #[tokio::test] + async fn bounded_scan_prefers_newer_over_stale_pointer() { + let env = TestEnv::new(10).await; + // Simulate: pointer was set to 3 a while ago, but mapping-sync has since + // indexed up to 8. The bounded scan must find 8, not return the stale 3. + for n in 1u64..=8 { + env.index_block(n); + } + env.set_pointer(3); + + let result = env.latest().await; + assert_eq!( + result, env.substrate_hashes[8], + "bounded scan must find block 8, not return stale pointer at 3" + ); + } + + #[tokio::test] + async fn reorg_with_stale_pointer_walks_past_stale_blocks() { + let env = TestEnv::new(5).await; + for n in 1u64..=3 { + env.index_block(n); + } + for n in 4u64..=5 { + env.write_stale_mapping(n); + } + env.set_pointer(5); + + let result = env.latest().await; + assert_ne!(result, env.genesis_hash(), "must not fall back to genesis"); + assert_eq!( + result, env.substrate_hashes[3], + "should return block 3 (highest valid indexed block)" + ); + } + + #[tokio::test] + async fn no_pointer_still_finds_indexed_blocks() { + let env = TestEnv::new(5).await; + for n in 1u64..=3 { + env.index_block(n); + } + // No pointer set — simulates DB corruption or first run after pointer loss. + + let result = env.latest().await; + assert_eq!( + result, env.substrate_hashes[3], + "should find block 3 via bounded scan even without a pointer" + ); + } + + #[tokio::test] + async fn initial_sync_nothing_indexed_returns_genesis() { + let env = TestEnv::new(5).await; + // No blocks indexed, no pointer. + + let result = env.latest().await; + assert_eq!( + result, + env.genesis_hash(), + "should return genesis when nothing is indexed" + ); + } + + #[tokio::test] + async fn genesis_only_returns_genesis() { + let env = TestEnv::new(0).await; + + let result = env.latest().await; + assert_eq!( + result, + env.genesis_hash(), + "should return genesis when chain is at block 0" + ); + } + + #[tokio::test] + async fn exhaustive_scan_finds_indexed_block_beyond_bounded_range() { + // With the test scan limit (8), best=20 yields: + // - bounded scan over [12..19] + // - exhaustive scan over [0..11] + let env = TestEnv::new(20).await; + env.index_block(5); + + let result = env.latest().await; + assert_eq!( + result, env.substrate_hashes[5], + "should recover block 5 via exhaustive scan when bounded scan misses" + ); + } + + #[tokio::test] + async fn persisted_pointer_used_when_both_scan_layers_miss() { + // With the test scan limit (8), best=40 covers: + // - bounded [32..39] + // - exhaustive [8..31] + // So block 3 is only reachable via the persisted pointer fallback. + let env = TestEnv::new(40).await; + env.index_block(3); + env.set_pointer(3); + + let result = env.latest().await; + assert_eq!( + result, env.substrate_hashes[3], + "should use persisted pointer when bounded+exhaustive scans both miss" + ); + } + + #[tokio::test] + async fn deep_recovery_finds_indexed_block_when_pointer_missing() { + // With scan limit 8: bounded [32..39], exhaustive [8..31]. Block 3 is outside both. + // Layer 3 deep recovery [0..7] finds it when no pointer exists. + let env = TestEnv::new(40).await; + env.index_block(3); + + let result = env.latest().await; + assert_ne!(result, env.genesis_hash(), "must not fall back to genesis"); + assert_eq!( + result, env.substrate_hashes[3], + "deep recovery should find block 3 when pointer is missing" + ); + } + + #[tokio::test] + async fn pointer_above_best_ignored_deep_recovery_finds_block() { + // Pointer corruption: pointer > best is ignored. Deep recovery should still + // find indexed block 3 in [0..7] when bounded+exhaustive miss. + let env = TestEnv::new(40).await; + env.index_block(3); + env.set_pointer(100); + + let result = env.latest().await; + assert_ne!(result, env.genesis_hash(), "must not fall back to genesis"); + assert_eq!( + result, env.substrate_hashes[3], + "deep recovery should find block 3 when pointer is invalid" + ); + } + + #[tokio::test] + async fn deep_recovery_preferred_over_stale_pointer() { + // Regression: stale-but-valid pointer (block 1) must not mask newer indexed block (3) + // in the deep range. Deep recovery [0..7] runs before pointer; must return block 3. + let env = TestEnv::new(40).await; + env.index_block(3); + env.set_pointer(1); + + let result = env.latest().await; + assert_eq!( + result, env.substrate_hashes[3], + "deep recovery must find block 3, not return older block 1 from pointer" + ); + } + + #[tokio::test] + async fn genesis_fallback_when_indexed_block_outside_all_windows() { + // With limit 8: deep recovery covers [best-96..best-33]. For best=100, [4..67]. + // Block 2 is outside; no pointer. Documents that genesis is still returned when + // indexed data exists but is beyond even the deep-recovery window. + let env = TestEnv::new(100).await; + env.index_block(2); + + let result = env.latest().await; + assert_eq!( + result, + env.genesis_hash(), + "indexed block outside all scan windows with no pointer yields genesis" + ); + } + + #[tokio::test] + async fn stale_pointer_target_backtracks_to_find_valid_block() { + // Pointer points to a stale/unusable number mapping (no canonical indexed + // block at that height). There is an older valid indexed block (2). The + // resolver must backtrack from pointer-1 and return block 2, not genesis. + let env = TestEnv::new(40).await; + env.index_block(2); + env.write_stale_mapping(3); + env.set_pointer(3); + + let result = env.latest().await; + assert_ne!(result, env.genesis_hash(), "must not fall back to genesis"); + assert_eq!( + result, env.substrate_hashes[2], + "should find block 2 via backtrack from stale pointer target" + ); + } + + #[tokio::test] + async fn pointer_updates_after_stale_pointer_backtrack_recovery() { + // After backtrack from a stale pointer, the persisted pointer should be + // updated to the block we found. + let env = TestEnv::new(40).await; + env.index_block(2); + env.write_stale_mapping(3); + env.set_pointer(3); + + let _ = env.latest().await; + + let updated = env + .backend + .mapping() + .latest_canonical_indexed_block_number() + .expect("read pointer"); + assert_eq!( + updated, + Some(2), + "pointer should be updated to block 2 found by backtrack" + ); + } + + #[tokio::test] + async fn pointer_updates_after_bounded_scan_recovery() { + let env = TestEnv::new(10).await; + for n in 1u64..=6 { + env.index_block(n); + } + env.set_pointer(3); + + let _ = env.latest().await; + + // After the call, the pointer should have been updated to 6. + let updated = env + .backend + .mapping() + .latest_canonical_indexed_block_number() + .expect("read pointer"); + assert_eq!( + updated, + Some(6), + "pointer should be updated to the block found by bounded scan" + ); + } +}