chore: remove total difficulty from HeaderProvider (#19151)

Author: joshieDo

crates/cli/commands/src/init_state/without_evm.rs

@@ -1,5 +1,5 @@
 use alloy_consensus::BlockHeader;
-use alloy_primitives::{BlockNumber, B256, U256};
+use alloy_primitives::{BlockNumber, B256};
 use alloy_rlp::Decodable;
 use reth_codecs::Compact;
 use reth_node_builder::NodePrimitives;
@@ -133,7 +133,7 @@ where
             for block_num in 1..=target_height {
                 // TODO: should we fill with real parent_hash?
                 let header = header_factory(block_num);
-                writer.append_header(&header, U256::ZERO, &B256::ZERO)?;
+                writer.append_header(&header, &B256::ZERO)?;
             }
             Ok(())
         });

crates/era-utils/src/export.rs

@@ -1,6 +1,7 @@
 //! Logic to export from database era1 block history
 //! and injecting them into era1 files with `Era1Writer`.
 
+use crate::calculate_td_by_number;
 use alloy_consensus::BlockHeader;
 use alloy_primitives::{BlockNumber, B256, U256};
 use eyre::{eyre, Result};
@@ -114,9 +115,7 @@ where
 
     let mut total_difficulty = if config.first_block_number > 0 {
         let prev_block_number = config.first_block_number - 1;
-        provider
-            .header_td_by_number(prev_block_number)?
-            .ok_or_else(|| eyre!("Total difficulty not found for block {prev_block_number}"))?
+        calculate_td_by_number(provider, prev_block_number)?
     } else {
         U256::ZERO
     };

crates/era-utils/src/history.rs

@@ -1,3 +1,4 @@
+use alloy_consensus::BlockHeader;
 use alloy_primitives::{BlockHash, BlockNumber, U256};
 use futures_util::{Stream, StreamExt};
 use reth_db_api::{
@@ -19,15 +20,15 @@ use reth_etl::Collector;
 use reth_fs_util as fs;
 use reth_primitives_traits::{Block, FullBlockBody, FullBlockHeader, NodePrimitives};
 use reth_provider::{
-    providers::StaticFileProviderRWRefMut, BlockWriter, ProviderError, StaticFileProviderFactory,
+    providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, StaticFileProviderFactory,
     StaticFileSegment, StaticFileWriter,
 };
 use reth_stages_types::{
     CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId,
 };
 use reth_storage_api::{
-    errors::ProviderResult, DBProvider, DatabaseProviderFactory, HeaderProvider,
-    NodePrimitivesProvider, StageCheckpointWriter,
+    errors::ProviderResult, DBProvider, DatabaseProviderFactory, NodePrimitivesProvider,
+    StageCheckpointWriter,
 };
 use std::{
     collections::Bound,
@@ -82,11 +83,6 @@ where
         .get_highest_static_file_block(StaticFileSegment::Headers)
         .unwrap_or_default();
 
-    // Find the latest total difficulty
-    let mut td = static_file_provider
-        .header_td_by_number(height)?
-        .ok_or(ProviderError::TotalDifficultyNotFound(height))?;
-
     while let Some(meta) = rx.recv()? {
         let from = height;
         let provider = provider_factory.database_provider_rw()?;
@@ -96,7 +92,6 @@ where
             &mut static_file_provider.latest_writer(StaticFileSegment::Headers)?,
             &provider,
             hash_collector,
-            &mut td,
             height..,
         )?;
 
@@ -146,7 +141,7 @@ where
 
 /// Extracts block headers and bodies from `meta` and appends them using `writer` and `provider`.
 ///
-/// Adds on to `total_difficulty` and collects hash to height using `hash_collector`.
+/// Collects hash to height using `hash_collector`.
 ///
 /// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the
 /// [`end_bound`] or the end of the file.
@@ -160,7 +155,6 @@ pub fn process<Era, P, B, BB, BH>(
     writer: &mut StaticFileProviderRWRefMut<'_, <P as NodePrimitivesProvider>::Primitives>,
     provider: &P,
     hash_collector: &mut Collector<BlockHash, BlockNumber>,
-    total_difficulty: &mut U256,
     block_numbers: impl RangeBounds<BlockNumber>,
 ) -> eyre::Result<BlockNumber>
 where
@@ -182,7 +176,7 @@ where
                 as Box<dyn Fn(Result<BlockTuple, E2sError>) -> eyre::Result<(BH, BB)>>);
     let iter = ProcessIter { iter, era: meta };
 
-    process_iter(iter, writer, provider, hash_collector, total_difficulty, block_numbers)
+    process_iter(iter, writer, provider, hash_collector, block_numbers)
 }
 
 type ProcessInnerIter<R, BH, BB> =
@@ -271,7 +265,6 @@ pub fn process_iter<P, B, BB, BH>(
     writer: &mut StaticFileProviderRWRefMut<'_, <P as NodePrimitivesProvider>::Primitives>,
     provider: &P,
     hash_collector: &mut Collector<BlockHash, BlockNumber>,
-    total_difficulty: &mut U256,
     block_numbers: impl RangeBounds<BlockNumber>,
 ) -> eyre::Result<BlockNumber>
 where
@@ -311,11 +304,8 @@ where
         let hash = header.hash_slow();
         last_header_number = number;
 
-        // Increase total difficulty
-        *total_difficulty += header.difficulty();
-
         // Append to Headers segment
-        writer.append_header(&header, *total_difficulty, &hash)?;
+        writer.append_header(&header, &hash)?;
 
         // Write bodies to database.
         provider.append_block_bodies(vec![(header.number(), Some(body))])?;
@@ -382,3 +372,28 @@ where
 
     Ok(())
 }
+
+/// Calculates the total difficulty for a given block number by summing the difficulty
+/// of all blocks from genesis to the given block.
+///
+/// Very expensive - iterates through all blocks in batches of 1000.
+///
+/// Returns an error if any block is missing.
+pub fn calculate_td_by_number<P>(provider: &P, num: BlockNumber) -> eyre::Result<U256>
+where
+    P: BlockReader,
+{
+    let mut total_difficulty = U256::ZERO;
+    let mut start = 0;
+
+    while start <= num {
+        let end = (start + 1000 - 1).min(num);
+
+        total_difficulty +=
+            provider.headers_range(start..=end)?.iter().map(|h| h.difficulty()).sum::<U256>();
+
+        start = end + 1;
+    }
+
+    Ok(total_difficulty)
+}

crates/era-utils/src/lib.rs

@@ -14,5 +14,6 @@ pub use export::{export, ExportConfig};
 
 /// Imports history from ERA files.
 pub use history::{
-    build_index, decode, import, open, process, process_iter, save_stage_checkpoints, ProcessIter,
+    build_index, calculate_td_by_number, decode, import, open, process, process_iter,
+    save_stage_checkpoints, ProcessIter,
 };

crates/net/downloaders/src/bodies/test_utils.rs

@@ -3,7 +3,7 @@
 #![allow(dead_code)]
 
 use alloy_consensus::BlockHeader;
-use alloy_primitives::{B256, U256};
+use alloy_primitives::B256;
 use reth_ethereum_primitives::BlockBody;
 use reth_network_p2p::bodies::response::BlockResponse;
 use reth_primitives_traits::{Block, SealedBlock, SealedHeader};
@@ -55,9 +55,7 @@ pub(crate) fn insert_headers(
         .expect("failed to create writer");
 
     for header in headers {
-        writer
-            .append_header(header.header(), U256::ZERO, &header.hash())
-            .expect("failed to append header");
+        writer.append_header(header.header(), &header.hash()).expect("failed to append header");
     }
     drop(writer);
     provider_rw.commit().expect("failed to commit");

crates/node/core/src/node_config.rs

@@ -10,7 +10,7 @@ use crate::{
 };
 use alloy_consensus::BlockHeader;
 use alloy_eips::BlockHashOrNumber;
-use alloy_primitives::{BlockNumber, B256};
+use alloy_primitives::{BlockNumber, B256, U256};
 use eyre::eyre;
 use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET};
 use reth_config::config::PruneConfig;
@@ -346,12 +346,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
             .header_by_number(head)?
             .expect("the header for the latest block is missing, database is corrupt");
 
-        let total_difficulty = provider
-            .header_td_by_number(head)?
-            // total difficulty is effectively deprecated, but still required in some places, e.g.
-            // p2p
-            .unwrap_or_default();
-
         let hash = provider
             .block_hash(head)?
             .expect("the hash for the latest block is missing, database is corrupt");
@@ -360,7 +354,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
             number: head,
             hash,
             difficulty: header.difficulty(),
-            total_difficulty,
+            total_difficulty: U256::ZERO,
             timestamp: header.timestamp(),
         })
     }

crates/rpc/rpc-eth-types/src/error/mod.rs

@@ -462,7 +462,6 @@ impl From<reth_errors::ProviderError> for EthApiError {
             }
             ProviderError::BestBlockNotFound => Self::HeaderNotFound(BlockId::latest()),
             ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex,
-            ProviderError::TotalDifficultyNotFound(num) => Self::HeaderNotFound(num.into()),
             ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()),
             ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()),
             err => Self::Internal(err.into()),

crates/stages/stages/src/stages/era.rs

@@ -10,12 +10,11 @@ use reth_era_utils as era;
 use reth_etl::Collector;
 use reth_primitives_traits::{FullBlockBody, FullBlockHeader, NodePrimitives};
 use reth_provider::{
-    BlockReader, BlockWriter, DBProvider, HeaderProvider, StageCheckpointWriter,
-    StaticFileProviderFactory, StaticFileWriter,
+    BlockReader, BlockWriter, DBProvider, StageCheckpointWriter, StaticFileProviderFactory,
+    StaticFileWriter,
 };
 use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
 use reth_static_file_types::StaticFileSegment;
-use reth_storage_errors::ProviderError;
 use std::{
     fmt::{Debug, Formatter},
     iter,
@@ -176,11 +175,6 @@ where
                 .get_highest_static_file_block(StaticFileSegment::Headers)
                 .unwrap_or_default();
 
-            // Find the latest total difficulty
-            let mut td = static_file_provider
-                .header_td_by_number(last_header_number)?
-                .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?;
-
             // Although headers were downloaded in reverse order, the collector iterates it in
             // ascending order
             let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
@@ -190,7 +184,6 @@ where
                 &mut writer,
                 provider,
                 &mut self.hash_collector,
-                &mut td,
                 last_header_number..=input.target(),
             )
             .map_err(|e| StageError::Fatal(e.into()))?;
@@ -336,7 +329,7 @@ mod tests {
         };
         use reth_ethereum_primitives::TransactionSigned;
         use reth_primitives_traits::{SealedBlock, SealedHeader};
-        use reth_provider::{BlockNumReader, TransactionsProvider};
+        use reth_provider::{BlockNumReader, HeaderProvider, TransactionsProvider};
         use reth_testing_utils::generators::{
             random_block_range, random_signed_tx, BlockRangeParams,
         };
@@ -447,9 +440,6 @@ mod tests {
                 match output {
                     Some(output) if output.checkpoint.block_number > initial_checkpoint => {
                         let provider = self.db.factory.provider()?;
-                        let mut td = provider
-                            .header_td_by_number(initial_checkpoint.saturating_sub(1))?
-                            .unwrap_or_default();
 
                         for block_num in initial_checkpoint..
                             output
@@ -469,10 +459,6 @@ mod tests {
                             assert!(header.is_some());
                             let header = SealedHeader::seal_slow(header.unwrap());
                             assert_eq!(header.hash(), hash);
-
-                            // validate the header total difficulty
-                            td += header.difficulty;
-                            assert_eq!(provider.header_td_by_number(block_num)?, Some(td));
                         }
 
                         self.validate_db_blocks(

crates/stages/stages/src/stages/headers.rs

@@ -16,15 +16,14 @@ use reth_network_p2p::headers::{
 };
 use reth_primitives_traits::{serde_bincode_compat, FullBlockHeader, NodePrimitives, SealedHeader};
 use reth_provider::{
-    providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderProvider,
-    HeaderSyncGapProvider, StaticFileProviderFactory,
+    providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderSyncGapProvider,
+    StaticFileProviderFactory,
 };
 use reth_stages_api::{
     CheckpointBlockRange, EntitiesCheckpoint, ExecInput, ExecOutput, HeadersCheckpoint, Stage,
     StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput,
 };
 use reth_static_file_types::StaticFileSegment;
-use reth_storage_errors::provider::ProviderError;
 use std::task::{ready, Context, Poll};
 
 use tokio::sync::watch;
@@ -107,11 +106,6 @@ where
             .get_highest_static_file_block(StaticFileSegment::Headers)
             .unwrap_or_default();
 
-        // Find the latest total difficulty
-        let mut td = static_file_provider
-            .header_td_by_number(last_header_number)?
-            .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?;
-
         // Although headers were downloaded in reverse order, the collector iterates it in ascending
         // order
         let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
@@ -134,11 +128,8 @@ where
             }
             last_header_number = header.number();
 
-            // Increase total difficulty
-            td += header.difficulty();
-
             // Append to Headers segment
-            writer.append_header(header, td, header_hash)?;
+            writer.append_header(header, header_hash)?;
         }
 
         info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index");
@@ -415,7 +406,7 @@ mod tests {
             ReverseHeadersDownloader, ReverseHeadersDownloaderBuilder,
         };
         use reth_network_p2p::test_utils::{TestHeaderDownloader, TestHeadersClient};
-        use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader};
+        use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader, HeaderProvider};
         use tokio::sync::watch;
 
         pub(crate) struct HeadersTestRunner<D: HeaderDownloader> {
@@ -493,9 +484,6 @@ mod tests {
                 match output {
                     Some(output) if output.checkpoint.block_number > initial_checkpoint => {
                         let provider = self.db.factory.provider()?;
-                        let mut td = provider
-                            .header_td_by_number(initial_checkpoint.saturating_sub(1))?
-                            .unwrap_or_default();
 
                         for block_num in initial_checkpoint..output.checkpoint.block_number {
                             // look up the header hash
@@ -509,10 +497,6 @@ mod tests {
                             assert!(header.is_some());
                             let header = SealedHeader::seal_slow(header.unwrap());
                             assert_eq!(header.hash(), hash);
-
-                            // validate the header total difficulty
-                            td += header.difficulty;
-                            assert_eq!(provider.header_td_by_number(block_num)?, Some(td));
                         }
                     }
                     _ => self.check_no_header_entry_above(initial_checkpoint)?,
@@ -635,16 +619,7 @@ mod tests {
         let static_file_provider = provider.static_file_provider();
         let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap();
         for header in sealed_headers {
-            let ttd = if header.number() == 0 {
-                header.difficulty()
-            } else {
-                let parent_block_number = header.number() - 1;
-                let parent_ttd =
-                    provider.header_td_by_number(parent_block_number).unwrap().unwrap_or_default();
-                parent_ttd + header.difficulty()
-            };
-
-            writer.append_header(header.header(), ttd, &header.hash()).unwrap();
+            writer.append_header(header.header(), &header.hash()).unwrap();
         }
         drop(writer);
 

crates/stages/stages/src/stages/merkle.rs

@@ -738,7 +738,7 @@ mod tests {
             let hash = last_header.hash_slow();
             writer.prune_headers(1).unwrap();
             writer.commit().unwrap();
-            writer.append_header(&last_header, U256::ZERO, &hash).unwrap();
+            writer.append_header(&last_header, &hash).unwrap();
             writer.commit().unwrap();
 
             Ok(blocks)