Merge #1034: Implement linked-list LocalChain and update chain-src crates/examples

b206a98 fix: Even more refactoring to code and documentation (志宇)
bea8e5a fix: `TxGraph::missing_blocks` logic (志宇)
db15e03 fix: improve more docs and more refactoring (志宇)
95312d4 fix: docs and some minor refactoring (志宇)
8bf7a99 Refactor `debug_assertions` checks for `LocalChain` (志宇)
315e7e0 fix: rm duplicate `bdk_tmp_plan` module (志宇)
af705da Add exclusion of example cli `*.db` files in `.gitignore` (志宇)
eabeb6c Implement linked-list `LocalChain` and update chain-src crates/examples (志宇)

Pull request description:

  Fixes #997
  Replaces #1002

  ### Description

  This PR changes the `LocalChain` implementation to have blocks stored as a linked-list. This allows the data-src thread to hold a shared ref to a single checkpoint and have access to the whole history of checkpoints without cloning or keeping a lock on `LocalChain`.

  The APIs of `bdk::Wallet`, `esplora` and `electrum` are also updated to reflect these changes. Note that the `esplora` crate is rewritten to anchor txs in the confirmation block (using the esplora API's tx status block_hash). This guarantees 100% consistency between anchor blocks and their transactions (instead of anchoring txs to the latest tip). `ExploraExt` now has separate methods for updating the `TxGraph` and `LocalChain`.

  A new method `TxGraph::missing_blocks` is introduced for finding "floating anchors" of a `TxGraph` update (given a chain).

  Additional changes:

  * `test_local_chain.rs` is refactored to make test cases easier to write. Additional tests are also added.
  * Examples are updated.
  * Exclude example-cli `*.db` files in `.gitignore`.
  * Rm duplicate `bdk_tmp_plan` module.

  ### Notes to the reviewers

  This is the smallest possible division of #1002 without resulting in PRs that do not compile. Since we have changed the API of `LocalChain`, we also need to change `esplora`, `electrum` crates and examples alongside `bdk::Wallet`.

  ### Changelog notice

  * Implement linked-list `LocalChain`. This allows the data-src thread to hold a shared ref to a single checkpoint and have access to the whole history of checkpoints without cloning or keeping a lock on `LocalChain`.
  * Rewrote `esplora` chain-src crate to anchor txs to their confirmation blocks (using esplora API's tx-status `block_hash`).

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature

ACKs for top commit:
  LLFourn:
    ACK b206a98

Tree-SHA512: a513eecb4f1aae6a5c06a69854e4492961424312a75a42d74377d363b364e3d52415bc81b4aa3fbc3f369ded19bddd07ab895130ebba288e8a43e9d6186e9fcc

Author: evanlinjin

.gitignore

@@ -4,3 +4,6 @@ Cargo.lock
 
 *.swp
 .idea
+
+# Example persisted files.
+*.db

crates/bdk/src/wallet/mod.rs

@@ -23,7 +23,7 @@ pub use bdk_chain::keychain::Balance;
 use bdk_chain::{
     indexed_tx_graph::IndexedAdditions,
     keychain::{KeychainTxOutIndex, LocalChangeSet, LocalUpdate},
-    local_chain::{self, LocalChain, UpdateNotConnectedError},
+    local_chain::{self, CannotConnectError, CheckPoint, CheckPointIter, LocalChain},
     tx_graph::{CanonicalTx, TxGraph},
     Append, BlockId, ChainPosition, ConfirmationTime, ConfirmationTimeAnchor, FullTxOut,
     IndexedTxGraph, Persist, PersistBackend,
@@ -32,8 +32,8 @@ use bitcoin::consensus::encode::serialize;
 use bitcoin::secp256k1::Secp256k1;
 use bitcoin::util::psbt;
 use bitcoin::{
-    Address, BlockHash, EcdsaSighashType, LockTime, Network, OutPoint, SchnorrSighashType, Script,
-    Sequence, Transaction, TxOut, Txid, Witness,
+    Address, EcdsaSighashType, LockTime, Network, OutPoint, SchnorrSighashType, Script, Sequence,
+    Transaction, TxOut, Txid, Witness,
 };
 use core::fmt;
 use core::ops::Deref;
@@ -245,7 +245,7 @@ impl<D> Wallet<D> {
         };
 
         let changeset = db.load_from_persistence().map_err(NewError::Persist)?;
-        chain.apply_changeset(changeset.chain_changeset);
+        chain.apply_changeset(&changeset.chain_changeset);
         indexed_graph.apply_additions(changeset.indexed_additions);
 
         let persist = Persist::new(db);
@@ -370,19 +370,19 @@ impl<D> Wallet<D> {
             .graph()
             .filter_chain_unspents(
                 &self.chain,
-                self.chain.tip().unwrap_or_default(),
+                self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default(),
                 self.indexed_graph.index.outpoints().iter().cloned(),
             )
             .map(|((k, i), full_txo)| new_local_utxo(k, i, full_txo))
     }
 
     /// Get all the checkpoints the wallet is currently storing indexed by height.
-    pub fn checkpoints(&self) -> &BTreeMap<u32, BlockHash> {
-        self.chain.blocks()
+    pub fn checkpoints(&self) -> CheckPointIter {
+        self.chain.iter_checkpoints()
     }
 
     /// Returns the latest checkpoint.
-    pub fn latest_checkpoint(&self) -> Option<BlockId> {
+    pub fn latest_checkpoint(&self) -> Option<CheckPoint> {
         self.chain.tip()
     }
 
@@ -420,7 +420,7 @@ impl<D> Wallet<D> {
             .graph()
             .filter_chain_unspents(
                 &self.chain,
-                self.chain.tip().unwrap_or_default(),
+                self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default(),
                 core::iter::once((spk_i, op)),
             )
             .map(|((k, i), full_txo)| new_local_utxo(k, i, full_txo))
@@ -437,7 +437,7 @@ impl<D> Wallet<D> {
         let canonical_tx = CanonicalTx {
             observed_as: graph.get_chain_position(
                 &self.chain,
-                self.chain.tip().unwrap_or_default(),
+                self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default(),
                 txid,
             )?,
             node: graph.get_tx_node(txid)?,
@@ -460,7 +460,7 @@ impl<D> Wallet<D> {
     pub fn insert_checkpoint(
         &mut self,
         block_id: BlockId,
-    ) -> Result<bool, local_chain::InsertBlockNotMatchingError>
+    ) -> Result<bool, local_chain::InsertBlockError>
     where
         D: PersistBackend<ChangeSet>,
     {
@@ -504,13 +504,13 @@ impl<D> Wallet<D> {
                     .range(height..)
                     .next()
                     .ok_or(InsertTxError::ConfirmationHeightCannotBeGreaterThanTip {
-                        tip_height: self.chain.tip().map(|b| b.height),
+                        tip_height: self.chain.tip().map(|b| b.height()),
                         tx_height: height,
                     })
-                    .map(|(&anchor_height, &anchor_hash)| ConfirmationTimeAnchor {
+                    .map(|(&anchor_height, &hash)| ConfirmationTimeAnchor {
                         anchor_block: BlockId {
                             height: anchor_height,
-                            hash: anchor_hash,
+                            hash,
                         },
                         confirmation_height: height,
                         confirmation_time: time,
@@ -531,17 +531,18 @@ impl<D> Wallet<D> {
     pub fn transactions(
         &self,
     ) -> impl Iterator<Item = CanonicalTx<'_, Transaction, ConfirmationTimeAnchor>> + '_ {
-        self.indexed_graph
-            .graph()
-            .list_chain_txs(&self.chain, self.chain.tip().unwrap_or_default())
+        self.indexed_graph.graph().list_chain_txs(
+            &self.chain,
+            self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default(),
+        )
     }
 
     /// Return the balance, separated into available, trusted-pending, untrusted-pending and immature
     /// values.
     pub fn get_balance(&self) -> Balance {
         self.indexed_graph.graph().balance(
             &self.chain,
-            self.chain.tip().unwrap_or_default(),
+            self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default(),
             self.indexed_graph.index.outpoints().iter().cloned(),
             |&(k, _), _| k == KeychainKind::Internal,
         )
@@ -715,8 +716,7 @@ impl<D> Wallet<D> {
             None => self
                 .chain
                 .tip()
-                .and_then(|cp| cp.height.into())
-                .map(|height| LockTime::from_height(height).expect("Invalid height")),
+                .map(|cp| LockTime::from_height(cp.height()).expect("Invalid height")),
             h => h,
         };
 
@@ -1030,7 +1030,7 @@ impl<D> Wallet<D> {
     ) -> Result<TxBuilder<'_, D, DefaultCoinSelectionAlgorithm, BumpFee>, Error> {
         let graph = self.indexed_graph.graph();
         let txout_index = &self.indexed_graph.index;
-        let chain_tip = self.chain.tip().unwrap_or_default();
+        let chain_tip = self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default();
 
         let mut tx = graph
             .get_tx(txid)
@@ -1265,7 +1265,7 @@ impl<D> Wallet<D> {
         psbt: &mut psbt::PartiallySignedTransaction,
         sign_options: SignOptions,
     ) -> Result<bool, Error> {
-        let chain_tip = self.chain.tip().unwrap_or_default();
+        let chain_tip = self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default();
 
         let tx = &psbt.unsigned_tx;
         let mut finished = true;
@@ -1288,7 +1288,7 @@ impl<D> Wallet<D> {
                 });
             let current_height = sign_options
                 .assume_height
-                .or(self.chain.tip().map(|b| b.height));
+                .or(self.chain.tip().map(|b| b.height()));
 
             debug!(
                 "Input #{} - {}, using `confirmation_height` = {:?}, `current_height` = {:?}",
@@ -1433,7 +1433,7 @@ impl<D> Wallet<D> {
         must_only_use_confirmed_tx: bool,
         current_height: Option<u32>,
     ) -> (Vec<WeightedUtxo>, Vec<WeightedUtxo>) {
-        let chain_tip = self.chain.tip().unwrap_or_default();
+        let chain_tip = self.chain.tip().map(|cp| cp.block_id()).unwrap_or_default();
         //    must_spend <- manually selected utxos
         //    may_spend  <- all other available utxos
         let mut may_spend = self.get_available_utxos();
@@ -1698,27 +1698,26 @@ impl<D> Wallet<D> {
 
     /// Applies an update to the wallet and stages the changes (but does not [`commit`] them).
     ///
-    /// This returns whether the `update` resulted in any changes.
-    ///
     /// Usually you create an `update` by interacting with some blockchain data source and inserting
     /// transactions related to your wallet into it.
     ///
     /// [`commit`]: Self::commit
-    pub fn apply_update(&mut self, update: Update) -> Result<bool, UpdateNotConnectedError>
+    pub fn apply_update(&mut self, update: Update) -> Result<(), CannotConnectError>
     where
         D: PersistBackend<ChangeSet>,
     {
-        let mut changeset: ChangeSet = self.chain.apply_update(update.chain)?.into();
+        let mut changeset = ChangeSet::from(self.chain.apply_update(update.chain)?);
         let (_, index_additions) = self
             .indexed_graph
             .index
-            .reveal_to_target_multi(&update.keychain);
+            .reveal_to_target_multi(&update.last_active_indices);
         changeset.append(ChangeSet::from(IndexedAdditions::from(index_additions)));
-        changeset.append(self.indexed_graph.apply_update(update.graph).into());
+        changeset.append(ChangeSet::from(
+            self.indexed_graph.apply_update(update.graph),
+        ));
 
-        let changed = !changeset.is_empty();
         self.persist.stage(changeset);
-        Ok(changed)
+        Ok(())
     }
 
     /// Commits all curently [`staged`] changed to the persistence backend returning and error when

crates/bdk/tests/wallet.rs

@@ -44,7 +44,10 @@ fn receive_output(wallet: &mut Wallet, value: u64, height: ConfirmationTime) ->
 
 fn receive_output_in_latest_block(wallet: &mut Wallet, value: u64) -> OutPoint {
     let height = match wallet.latest_checkpoint() {
-        Some(BlockId { height, .. }) => ConfirmationTime::Confirmed { height, time: 0 },
+        Some(cp) => ConfirmationTime::Confirmed {
+            height: cp.height(),
+            time: 0,
+        },
         None => ConfirmationTime::Unconfirmed { last_seen: 0 },
     };
     receive_output(wallet, value, height)
@@ -222,7 +225,7 @@ fn test_create_tx_fee_sniping_locktime_last_sync() {
     // If there's no current_height we're left with using the last sync height
     assert_eq!(
         psbt.unsigned_tx.lock_time.0,
-        wallet.latest_checkpoint().unwrap().height
+        wallet.latest_checkpoint().unwrap().height()
     );
 }
 
@@ -426,11 +429,7 @@ fn test_create_tx_drain_wallet_and_drain_to_and_with_recipient() {
 fn test_create_tx_drain_to_and_utxos() {
     let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
     let addr = wallet.get_address(New);
-    let utxos: Vec<_> = wallet
-        .list_unspent()
-        .into_iter()
-        .map(|u| u.outpoint)
-        .collect();
+    let utxos: Vec<_> = wallet.list_unspent().map(|u| u.outpoint).collect();
     let mut builder = wallet.build_tx();
     builder
         .drain_to(addr.script_pubkey())
@@ -1482,7 +1481,7 @@ fn test_bump_fee_drain_wallet() {
         .insert_tx(
             tx.clone(),
             ConfirmationTime::Confirmed {
-                height: wallet.latest_checkpoint().unwrap().height,
+                height: wallet.latest_checkpoint().unwrap().height(),
                 time: 42_000,
             },
         )

crates/chain/src/keychain.rs

@@ -11,10 +11,7 @@
 //! [`SpkTxOutIndex`]: crate::SpkTxOutIndex
 
 use crate::{
-    collections::BTreeMap,
-    indexed_tx_graph::IndexedAdditions,
-    local_chain::{self, LocalChain},
-    tx_graph::TxGraph,
+    collections::BTreeMap, indexed_tx_graph::IndexedAdditions, local_chain, tx_graph::TxGraph,
     Anchor, Append,
 };
 
@@ -85,24 +82,33 @@ impl<K> AsRef<BTreeMap<K, u32>> for DerivationAdditions<K> {
     }
 }
 
-/// A structure to update [`KeychainTxOutIndex`], [`TxGraph`] and [`LocalChain`]
-/// atomically.
-#[derive(Debug, Clone, PartialEq)]
+/// A structure to update [`KeychainTxOutIndex`], [`TxGraph`] and [`LocalChain`] atomically.
+///
+/// [`LocalChain`]: local_chain::LocalChain
+#[derive(Debug, Clone)]
 pub struct LocalUpdate<K, A> {
-    /// Last active derivation index per keychain (`K`).
-    pub keychain: BTreeMap<K, u32>,
+    /// Contains the last active derivation indices per keychain (`K`), which is used to update the
+    /// [`KeychainTxOutIndex`].
+    pub last_active_indices: BTreeMap<K, u32>,
+
     /// Update for the [`TxGraph`].
     pub graph: TxGraph<A>,
+
     /// Update for the [`LocalChain`].
-    pub chain: LocalChain,
+    ///
+    /// [`LocalChain`]: local_chain::LocalChain
+    pub chain: local_chain::Update,
 }
 
-impl<K, A> Default for LocalUpdate<K, A> {
-    fn default() -> Self {
+impl<K, A> LocalUpdate<K, A> {
+    /// Construct a [`LocalUpdate`] with a given [`local_chain::Update`].
+    ///
+    /// [`CheckPoint`]: local_chain::CheckPoint
+    pub fn new(chain_update: local_chain::Update) -> Self {
         Self {
-            keychain: Default::default(),
-            graph: Default::default(),
-            chain: Default::default(),
+            last_active_indices: BTreeMap::new(),
+            graph: TxGraph::default(),
+            chain: chain_update,
         }
     }
 }
@@ -122,6 +128,8 @@ impl<K, A> Default for LocalUpdate<K, A> {
 )]
 pub struct LocalChangeSet<K, A> {
     /// Changes to the [`LocalChain`].
+    ///
+    /// [`LocalChain`]: local_chain::LocalChain
     pub chain_changeset: local_chain::ChangeSet,
 
     /// Additions to [`IndexedTxGraph`].