From ce8daaa7d4c046572728770cbe0dba40c6c848e3 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Tue, 18 Aug 2026 14:27:39 -0500 Subject: [PATCH 1/4] Only adopt a funding payment's own transactions from wallet sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wallet sync resolves a funding payment's id for any transaction linked to the record through its conflicting txids, and then adopted that transaction's txid and confirmation outright. A cooperative close conflicts with a pending splice in exactly that way: the splice record would report the close's txid and confirmation under its InteractiveFunding type and contribution figures and graduate as if the splice had confirmed, while the close's own record never received its confirmation. Adopt a transaction only when it is part of the payment's funding history — the record's current txid or a classified candidate. Anything else is recorded under its own txid-keyed id, which also delivers the close's confirmation to the close's own record. Generated with assistance from Claude Code. Co-Authored-By: Claude Fable 5 --- src/wallet/mod.rs | 189 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 164 insertions(+), 25 deletions(-) diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index b9c12b4a7..f8dd13db1 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -346,12 +346,12 @@ impl Wallet { // duplicating) the record classification just wrote. let guard = self.funding_payment_update_lock.lock().await; - let payment_id = self + let mut payment_id = self .find_payment_by_txid(txid) .await? .unwrap_or_else(|| PaymentId(txid.to_byte_array())); - if self + match self .apply_funding_status_update_locked( &guard, payment_id, @@ -360,7 +360,13 @@ impl Wallet { ) .await? { - continue; + FundingStatusUpdate::Applied => continue, + FundingStatusUpdate::NotFunding => {}, + // Not part of the funding payment's history (e.g. a close spending the + // funding outpoint): record it under its own id below instead. + FundingStatusUpdate::Foreign => { + payment_id = PaymentId(txid.to_byte_array()); + }, } let payment = { @@ -487,12 +493,12 @@ impl Wallet { // with classification. let guard = self.funding_payment_update_lock.lock().await; - let payment_id = self + let mut payment_id = self .find_payment_by_txid(txid) .await? .unwrap_or_else(|| PaymentId(txid.to_byte_array())); - if self + match self .apply_funding_status_update_locked( &guard, payment_id, @@ -501,7 +507,13 @@ impl Wallet { ) .await? { - continue; + FundingStatusUpdate::Applied => continue, + FundingStatusUpdate::NotFunding => {}, + // Not part of the funding payment's history (e.g. a close spending the + // funding outpoint): record it under its own id below instead. + FundingStatusUpdate::Foreign => { + payment_id = PaymentId(txid.to_byte_array()); + }, } let payment = { @@ -563,12 +575,12 @@ impl Wallet { // with classification. let guard = self.funding_payment_update_lock.lock().await; - let payment_id = self + let mut payment_id = self .find_payment_by_txid(txid) .await? .unwrap_or_else(|| PaymentId(txid.to_byte_array())); - if self + match self .apply_funding_status_update_locked( &guard, payment_id, @@ -577,7 +589,13 @@ impl Wallet { ) .await? { - continue; + FundingStatusUpdate::Applied => continue, + FundingStatusUpdate::NotFunding => {}, + // Not part of the funding payment's history (e.g. a close spending the + // funding outpoint): record it under its own id below instead. + FundingStatusUpdate::Foreign => { + payment_id = PaymentId(txid.to_byte_array()); + }, } let payment = { @@ -1938,9 +1956,11 @@ impl Wallet { /// If `payment_id` refers to a classified funding payment, refreshes its confirmation status /// and the candidate txid the event refers to, while preserving the contribution-derived /// amount/fee and `tx_type` that wallet sync must not recompute from its own view: the wallet's - /// `sent`/`received` don't capture our contribution to a shared funding output. Returns `true` - /// when it handled the payment, so the caller skips the default on-chain path. Graduation to - /// `Succeeded` is left to `ChainTipChanged` after `ANTI_REORG_DELAY`. + /// `sent`/`received` don't capture our contribution to a shared funding output. Returns + /// [`FundingStatusUpdate::Applied`] when it handled the payment, so the caller skips the + /// default on-chain path — or [`FundingStatusUpdate::Foreign`] when the transaction is not + /// part of the payment's funding history, so the caller records it under its own id. + /// Graduation to `Succeeded` is left to `ChainTipChanged` after `ANTI_REORG_DELAY`. /// /// The caller must hold [`Self::funding_payment_update_lock`] — from resolving `payment_id` /// through its own last write, not just across this call — so that classification's two-store @@ -1949,38 +1969,51 @@ impl Wallet { async fn apply_funding_status_update_locked( &self, _guard: &tokio::sync::MutexGuard<'_, ()>, payment_id: PaymentId, event_txid: Txid, confirmation_status: ConfirmationStatus, - ) -> Result { + ) -> Result { // The caller's wallet-level lock keeps the candidate history stable while we await its - // read. The funding-type gate and write then share the payment store's mutation lock: - // against a separate payment `get`, a classification merging in between would have its - // `tx_type` and contribution figures clobbered by this stale snapshot. + // read. The funding-type gate, the candidate lookup, and the write then share the payment + // store's mutation lock: against a separate payment `get`, a classification merging in + // between would have its `tx_type` and contribution figures clobbered by this stale + // snapshot. let pending_payment = self.pending_payment_store.get(&payment_id).await?; + let mut outcome = FundingStatusUpdate::NotFunding; let mut handled = None; self.payment_store .mutate(&payment_id, |existing| { let payment = existing?; - let tx_type = match &payment.kind { + let (current_txid, tx_type) = match &payment.kind { PaymentKind::Onchain { + txid, tx_type: tx_type @ Some( TransactionType::Funding { .. } | TransactionType::InteractiveFunding { .. }, ), .. - } => tx_type.clone(), + } => (*txid, tx_type.clone()), _ => return None, }; + // Adopt the event's txid only when the transaction is part of this payment's + // funding history: its current txid or a classified candidate. A conflicting + // transaction that is neither — a close also spends the funding outpoint — must + // not overwrite the record. + let owns_event_tx = event_txid == current_txid + || pending_payment.as_ref().is_some_and(|p| p.candidate(event_txid).is_some()); + if !owns_event_tx { + outcome = FundingStatusUpdate::Foreign; + return None; + } // Report the figures of the candidate that actually confirmed, which need not be // the last one broadcast (an earlier, lower-fee candidate may win) and may carry // no figures at all (`None`) for a round we didn't contribute to. (`direction` is // invariant across a splice's candidates and cannot be changed through the store // anyway.) let mut target = payment.clone(); - if let Some(pending) = pending_payment.as_ref() { - if let Some(candidate) = pending.candidate(event_txid) { - target.amount_msat = candidate.amount_msat; - target.fee_paid_msat = candidate.fee_paid_msat; - } + if let Some(candidate) = + pending_payment.as_ref().and_then(|p| p.candidate(event_txid)) + { + target.amount_msat = candidate.amount_msat; + target.fee_paid_msat = candidate.fee_paid_msat; } target.kind = PaymentKind::Onchain { txid: event_txid, status: confirmation_status, tx_type }; @@ -1998,7 +2031,7 @@ impl Wallet { }) .await?; let Some(payment) = handled else { - return Ok(false); + return Ok(outcome); }; // Mirror the refreshed confirmation status onto the pending entry: `ChainTipChanged` // graduates by reading the pending entry's details, so it must see the new status. This is @@ -2008,7 +2041,7 @@ impl Wallet { let pending = self.create_pending_payment_from_tx(payment, Vec::new()); self.pending_payment_store.insert_or_update(pending).await?; } - Ok(true) + Ok(FundingStatusUpdate::Applied) } #[allow(deprecated)] @@ -2311,6 +2344,20 @@ fn aggregate_local_stakes(candidate: &FundingCandidate) -> LocalStakeAggregate { } } +/// The outcome of [`Wallet::apply_funding_status_update_locked`]. +enum FundingStatusUpdate { + /// The event's transaction belongs to the funding payment; its refreshed confirmation status + /// was applied (or was already current). + Applied, + /// The resolved payment is not a classified funding payment; the caller's default on-chain + /// handling applies under the resolved id. + NotFunding, + /// The event's transaction is not part of the funding payment's history — e.g. a close + /// spending the same funding outpoint — so the funding record must not adopt it; the caller + /// should record the transaction under its own txid-derived id. + Foreign, +} + impl Listen for Wallet { fn filtered_block_connected( &self, _header: &bitcoin::block::Header, @@ -3960,6 +4007,98 @@ mod tests { assert_eq!(wallet.find_payment_by_txid(txid2).await.unwrap(), Some(payment_id)); } + /// A cooperative close conflicts with a pending splice's funding transaction — both spend the + /// pre-splice funding outpoint — so sync records the close among the splice record's + /// conflicting txids, and the close's confirmation then resolves to the splice's PaymentId. + /// The funding record must not adopt the close's txid and confirmation as its own: the close + /// is not a round of the splice. It must land on a record keyed by the close's own id. + #[tokio::test] + async fn funding_record_does_not_adopt_a_conflicting_close() { + let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let wallet = new_test_wallet(store, false).await; + + let funding_outpoint = + bitcoin::OutPoint { txid: Txid::from_byte_array([3u8; 32]), vout: 0 }; + + // The close pays the shutdown script, which is a wallet address. + let script_pubkey = wallet + .inner + .lock() + .unwrap() + .reveal_next_address(KeychainKind::External) + .address + .script_pubkey(); + let close_tx = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: LockTime::ZERO, + input: vec![bitcoin::TxIn { + previous_output: funding_outpoint, + script_sig: bitcoin::ScriptBuf::new(), + sequence: bitcoin::Sequence::MAX, + witness: bitcoin::Witness::new(), + }], + output: vec![TxOut { value: Amount::from_sat(90_000), script_pubkey }], + }; + let close_txid = close_tx.compute_txid(); + + let splice_txid = Txid::from_byte_array([2u8; 32]); + let payment_id = PaymentId([21u8; 32]); + let candidates = vec![FundingTxCandidate { + txid: splice_txid, + amount_msat: Some(1_000_000), + fee_paid_msat: Some(500), + }]; + let details = + interactive_funding_details(payment_id, splice_txid, Some(1_000_000), Some(500)); + wallet.persist_funding_payment(details, candidates).await.unwrap(); + + // Sync saw the close double-spend the splice's funding transaction. + wallet + .pending_payment_store + .update(PendingPaymentDetailsUpdate { + id: payment_id, + payment_update: None, + conflicting_txids: Some(vec![close_txid]), + candidates: Vec::new(), + }) + .await + .unwrap(); + + let event = WalletEvent::TxConfirmed { + txid: close_txid, + tx: Arc::new(close_tx), + block_time: confirmed_block_time(5), + old_block_time: None, + }; + wallet.update_payment_store(vec![event]).await.unwrap(); + + let funding = wallet.payment_store.get(&payment_id).await.unwrap().unwrap(); + match &funding.kind { + PaymentKind::Onchain { txid, status, tx_type } => { + assert_eq!(*txid, splice_txid, "the record must not adopt the close's txid"); + assert!(matches!(status, ConfirmationStatus::Unconfirmed)); + assert!(matches!(tx_type, Some(TransactionType::InteractiveFunding { .. }))); + }, + kind => panic!("unexpected kind {:?}", kind), + } + assert_eq!(funding.amount_msat, Some(1_000_000)); + assert_eq!(funding.fee_paid_msat, Some(500)); + + let close = wallet + .payment_store + .get(&PaymentId(close_txid.to_byte_array())) + .await + .unwrap() + .unwrap(); + match &close.kind { + PaymentKind::Onchain { txid, status, .. } => { + assert_eq!(*txid, close_txid); + assert!(matches!(status, ConfirmationStatus::Confirmed { .. })); + }, + kind => panic!("unexpected kind {:?}", kind), + } + } + /// A funding-typed broadcast that doesn't touch the on-chain wallet must not be recorded. /// LDK re-broadcasts a promoted-but-unconfirmed 0conf splice through its generic funding /// path, so a splice the interactive-funding classification deliberately declined — no local From ebc0086cdad1ea31709acaca77541d4f93a00066 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Tue, 18 Aug 2026 17:07:12 -0500 Subject: [PATCH 2/4] Retry funding-broadcast classification instead of dropping it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A queued broadcast whose payment-record classification failed was dropped outright, on the theory that broadcasting a transaction we failed to record would leave it on-chain without a payment. For interactive funding that theory doesn't hold: the counterparty broadcasts the same transaction once the signature exchange completes, so dropping the package keeps nothing off-chain — it only guarantees the round is never recorded as a candidate on our side. The funding-status ownership gate then treats the round's confirmation as foreign to the funding record and re-keys it to a stray duplicate record, which shadows the funding record's txid lookups permanently: the splice payment stays Pending forever while an untyped duplicate holds the confirmation. Keep the package alive instead: requeue it after a short delay and retry classification until it succeeds, holding the broadcast back the whole time. Classification failures are persistence failures, so the retry is unbounded — a store that never recovers keeps the node from functioning anyway — and every failed round is logged. Generated with assistance from Claude Code. Co-Authored-By: Claude Fable 5 --- src/chain/mod.rs | 29 ++++++++-------- src/tx_broadcaster.rs | 33 ++++++++++++++---- src/wallet/mod.rs | 81 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 21 deletions(-) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index f01c1c8cb..22151e9ec 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -578,20 +578,21 @@ impl ChainSource { } Some(next_package) = receiver.recv() => { // Classify funding broadcasts into payment records before sending. If - // classification fails we skip the broadcast, since broadcasting a tx we - // failed to record would leave it on-chain without a payment. - let package = match self.tx_broadcaster.classify_package(next_package).await { - Ok(package) => package, - Err(e) => { - log_error!( - tx_bcast_logger, - "Skipping broadcast: failed to persist payment records: {:?}", - e, - ); - continue; - }, - }; - let package = package.into_sorted_transactions(); + // classification fails we delay the broadcast and retry, since broadcasting + // a tx we failed to record would leave it on-chain without a payment — + // while dropping the package would not keep an interactively funded tx + // off-chain (the counterparty broadcasts it regardless), only leave it + // confirming without a recorded candidate. + if let Err(e) = self.tx_broadcaster.classify_package(&next_package).await { + log_error!( + tx_bcast_logger, + "Delaying broadcast: failed to persist payment records, will retry: {:?}", + e, + ); + self.tx_broadcaster.requeue_failed_classify(next_package); + continue; + } + let package = next_package.into_sorted_transactions(); match &self.kind { #[cfg(feature = "chain-esplora")] ChainSourceKind::Esplora(esplora_chain_source) => { diff --git a/src/tx_broadcaster.rs b/src/tx_broadcaster.rs index 782112dad..248926d45 100644 --- a/src/tx_broadcaster.rs +++ b/src/tx_broadcaster.rs @@ -7,6 +7,7 @@ use std::ops::Deref; use std::sync::{Mutex as StdMutex, Weak}; +use std::time::Duration; use bitcoin::Transaction; use lightning::chain::chaininterface::{ @@ -20,6 +21,11 @@ use crate::Error; const BCAST_PACKAGE_QUEUE_SIZE: usize = 256; +/// How long to wait before re-classifying a package whose classification failed. Long enough to +/// give a struggling store room to recover, short against the ~minutes until the transaction +/// could confirm. +const FAILED_CLASSIFY_RETRY_DELAY: Duration = Duration::from_secs(2); + /// A package of transactions that LDK handed to the broadcaster in one `broadcast_transactions` /// call, along with each transaction's type. Queued until the background task classifies and /// broadcasts it. Built only via [`BroadcastPackage::new`] from such a call, so unrelated @@ -133,12 +139,11 @@ where self.queue_receiver.lock().await } - /// Classifies a queued package into payment records and returns the package ready for the - /// chain client. Returns `Err` if any classification fails; callers must not broadcast the - /// package in that case, since a crash would leave the transaction on-chain without a record. - pub(crate) async fn classify_package( - &self, package: BroadcastPackage, - ) -> Result { + /// Classifies a queued package into payment records. Returns `Err` if any classification + /// fails; callers must not broadcast the package in that case, since a crash would leave the + /// transaction on-chain without a record — but must requeue it via + /// [`Self::requeue_failed_classify`] rather than drop it. + pub(crate) async fn classify_package(&self, package: &BroadcastPackage) -> Result<(), Error> { let wallet_opt = self.wallet.lock().expect("lock").as_ref().and_then(Weak::upgrade); if let Some(wallet) = wallet_opt { for (tx, tx_type) in package.transactions() { @@ -147,7 +152,21 @@ where } } } - Ok(package) + Ok(()) + } + + /// Re-sends a package whose classification failed back into the queue after a delay, so a + /// transient persistence failure delays the broadcast instead of dropping the package. + /// Dropping an interactive-funding package would not even keep its transaction off-chain — + /// the counterparty broadcasts it regardless — it would only leave the transaction + /// confirming without a recorded candidate. If the queue has closed by the time the delay + /// elapses, the node is shutting down and the package is dropped with it. + pub(crate) fn requeue_failed_classify(&self, package: BroadcastPackage) { + let sender = self.queue_sender.clone(); + tokio::spawn(async move { + tokio::time::sleep(FAILED_CLASSIFY_RETRY_DELAY).await; + let _ = sender.send(package).await; + }); } pub(crate) fn broadcast_unclassified_transaction(&self, tx: Transaction) { diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index f8dd13db1..6aa1300c8 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -4241,6 +4241,87 @@ mod tests { assert_unchanged(&wallet, payment_id, true).await; } + /// A funding broadcast whose classification fails must be retried, not dropped: for + /// interactive funding the counterparty broadcasts the same transaction regardless of + /// whether we do, so dropping the package permanently leaves the confirming transaction + /// unrecorded as a candidate — and the funding-status ownership gate then routes its + /// confirmation to a stray duplicate record instead of the funding record. + #[tokio::test] + async fn failed_funding_classification_is_retried_not_dropped() { + use lightning::chain::chaininterface::BroadcasterInterface; + + let fail_store = FailSwitchStore::new(); + let store: Arc = Arc::new(DynStoreWrapper(fail_store.clone())); + let wallet = new_test_wallet(Arc::clone(&store), false).await; + wallet.broadcaster.set_wallet(Arc::downgrade(&wallet)); + + // Run the production broadcast-queue loop. The broadcast itself fails fast against the + // fixture's unroutable Esplora server, which is irrelevant here: the record is written + // during classification, before the broadcast attempt. + let (stop_sender, stop_receiver) = tokio::sync::watch::channel(()); + let chain_source = Arc::clone(&wallet.chain_source); + let loop_task = tokio::spawn(async move { + chain_source.continuously_process_broadcast_queue(stop_receiver).await + }); + + // A funding transaction paying the wallet passes the wallet-activity guard, so its + // classification reaches the payment-store write. + let script_pubkey = wallet + .inner + .lock() + .unwrap() + .reveal_next_address(KeychainKind::External) + .address + .script_pubkey(); + let tx = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: LockTime::ZERO, + input: Vec::new(), + output: vec![TxOut { value: Amount::from_sat(10_000), script_pubkey }], + }; + let counterparty_node_id = PublicKey::from_str( + "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", + ) + .unwrap(); + + // Queue the broadcast while payment persistence is failing. + fail_store.fail_writes.store(true, Ordering::Release); + wallet.broadcaster.broadcast_transactions(&[( + &tx, + LdkTransactionType::Funding { + channels: vec![(counterparty_node_id, ChannelId([7u8; 32]))], + }, + )]); + + // Let the loop fail at least one classification round; a failed classification must not + // leave a partial record behind. + tokio::time::sleep(Duration::from_secs(3)).await; + assert!(wallet.payment_store.list_filter(|_| true).is_empty()); + + // Once writes recover, the package must still be alive to classify. + fail_store.fail_writes.store(false, Ordering::Release); + let mut recorded = Vec::new(); + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(100)).await; + recorded = wallet.payment_store.list_filter(|_| true); + if !recorded.is_empty() { + break; + } + } + assert!( + !recorded.is_empty(), + "the failed classification was never retried; the package was dropped" + ); + assert_eq!(recorded.len(), 1); + assert!(matches!( + recorded[0].kind, + PaymentKind::Onchain { tx_type: Some(TransactionType::Funding { .. }), .. } + )); + + stop_sender.send(()).unwrap(); + loop_task.await.unwrap(); + } + /// Barrier test, classification-first ordering: wallet sync's confirmation handling must /// wait for classification's two-store write pair. Classification is parked between its /// payment-store and pending-store writes (the torn window) and only then is the From bbcf5c4e689ddda0df48bf03b3c277425ba06e9f Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 26 Aug 2026 16:21:30 -0500 Subject: [PATCH 3/4] f - Wait for a failed write before letting the retry test recover The retry test slept a fixed three seconds and assumed classification had failed by then; if writes were re-enabled before the first attempt, the test would pass without any retry happening. Count failed writes in FailSwitchStore and wait for one before re-enabling writes. Also fix the test's store reads to use list_page: the payment store's cache is bounded, so list_filter is unavailable, and this commit did not compile its tests standalone (the conversion had landed in the following commit). Implemented with Claude Code. Co-Authored-By: Claude Fable 5 --- src/wallet/mod.rs | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 6aa1300c8..63c8719ae 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -2734,7 +2734,7 @@ fn funding_reclassification_update( #[cfg(all(test, any(feature = "chain-esplora", feature = "chain-electrum")))] mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use bdk_chain::{BlockId, ConfirmationBlockTime}; @@ -2764,11 +2764,13 @@ mod tests { const EXTERNAL_DESCRIPTOR: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)"; const INTERNAL_DESCRIPTOR: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)"; - /// An in-memory store whose writes can be made to fail on demand. + /// An in-memory store whose writes can be made to fail on demand, counting the failures so + /// tests can wait for a write to have actually failed rather than guessing with a sleep. #[derive(Clone)] struct FailSwitchStore { inner: Arc, fail_writes: Arc, + failed_writes: Arc, } impl FailSwitchStore { @@ -2776,6 +2778,7 @@ mod tests { Self { inner: Arc::new(InMemoryStore::new()), fail_writes: Arc::new(AtomicBool::new(false)), + failed_writes: Arc::new(AtomicUsize::new(0)), } } } @@ -2792,11 +2795,13 @@ mod tests { ) -> impl Future> + 'static + Send { let inner = Arc::clone(&self.inner); let fail_writes = Arc::clone(&self.fail_writes); + let failed_writes = Arc::clone(&self.failed_writes); let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); async move { if fail_writes.load(Ordering::Acquire) { + failed_writes.fetch_add(1, Ordering::AcqRel); return Err(io::Error::new(io::ErrorKind::Other, "writes disabled")); } KVStore::write(&*inner, &primary_namespace, &secondary_namespace, &key, buf).await @@ -4293,17 +4298,27 @@ mod tests { }, )]); - // Let the loop fail at least one classification round; a failed classification must not - // leave a partial record behind. - tokio::time::sleep(Duration::from_secs(3)).await; - assert!(wallet.payment_store.list_filter(|_| true).is_empty()); + // Wait until the loop has actually failed a classification write; re-enabling writes + // before the first attempt would let the first attempt succeed and the test pass + // without any retry happening. A failed classification must not leave a partial + // record behind. + let mut failed_writes = 0; + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(100)).await; + failed_writes = fail_store.failed_writes.load(Ordering::Acquire); + if failed_writes > 0 { + break; + } + } + assert!(failed_writes > 0, "classification never attempted a payment-store write"); + assert!(wallet.payment_store.list_page(None).await.unwrap().objects.is_empty()); // Once writes recover, the package must still be alive to classify. fail_store.fail_writes.store(false, Ordering::Release); let mut recorded = Vec::new(); for _ in 0..100 { tokio::time::sleep(Duration::from_millis(100)).await; - recorded = wallet.payment_store.list_filter(|_| true); + recorded = wallet.payment_store.list_page(None).await.unwrap().objects; if !recorded.is_empty() { break; } From fb85dd03e5d6d87c8baa93b9bfe0f091565b0fcd Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 26 Aug 2026 16:25:44 -0500 Subject: [PATCH 4/4] f - Keep classification retries inside the broadcast loop The retry for a failed classification was a detached tokio::spawn that outlived the node. Its comment claimed a re-send after shutdown would fail because the queue had closed, but the queue receiver lives in the broadcaster and is only dropped with the node, so the re-send succeeded and a stale package would be classified and broadcast after a stop()/start() cycle. Park failed packages inside the broadcast loop instead and retry them from a timer branch of the same select. New packages keep flowing while a retry waits, and pending retries are dropped when the loop stops. Implemented with Claude Code. Co-Authored-By: Claude Fable 5 --- src/chain/mod.rs | 86 +++++++++++++++++++++++++++++-------------- src/tx_broadcaster.rs | 23 +----------- src/wallet/mod.rs | 82 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 50 deletions(-) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 22151e9ec..947378d7e 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -37,9 +37,15 @@ use crate::config::{BackgroundSyncConfig, Config, WALLET_SYNC_INTERVAL_MINIMUM_S use crate::fee_estimator::OnchainFeeEstimator; use crate::logger::{log_debug, log_error, log_info, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; +use crate::tx_broadcaster::BroadcastPackage; use crate::types::{Broadcaster, ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; use crate::{Error, PersistedNodeMetrics}; +/// How long to wait before re-classifying a package whose classification failed. Long enough to +/// give a struggling store room to recover, short against the ~minutes until the transaction +/// could confirm. +const FAILED_CLASSIFY_RETRY_DELAY: Duration = Duration::from_secs(2); + /// We use this parent-child TRUC package to make sure the configured chain source supports /// broadcasting packages via the `submitpackage` Bitcoin Core RPC. const PARENT_TXID: &str = "9a015f93fac6cb203c2b994e18b85176eb0354a22a468255516f3c6002d3f696"; @@ -562,12 +568,53 @@ impl ChainSource { } } + /// Classifies the package's funding broadcasts into payment records, then broadcasts it. + /// Returns the package back on classification failure so the caller can retry it after a + /// delay: broadcasting a tx we failed to record would leave it on-chain without a payment, + /// while dropping the package would not keep an interactively funded tx off-chain (the + /// counterparty broadcasts it regardless), only leave it confirming without a recorded + /// candidate. + async fn classify_and_broadcast( + &self, package: BroadcastPackage, + ) -> Result<(), BroadcastPackage> { + if let Err(e) = self.tx_broadcaster.classify_package(&package).await { + log_error!( + self.logger, + "Delaying broadcast: failed to persist payment records, will retry: {:?}", + e, + ); + return Err(package); + } + let package = package.into_sorted_transactions(); + match &self.kind { + #[cfg(feature = "chain-esplora")] + ChainSourceKind::Esplora(esplora_chain_source) => { + esplora_chain_source.process_transaction_broadcast(package).await + }, + #[cfg(feature = "chain-electrum")] + ChainSourceKind::Electrum(electrum_chain_source) => { + electrum_chain_source.process_transaction_broadcast(package).await + }, + #[cfg(feature = "chain-bitcoind")] + ChainSourceKind::Bitcoind(bitcoind_chain_source) => { + bitcoind_chain_source.process_transaction_broadcast(package).await + }, + } + Ok(()) + } + pub(crate) async fn continuously_process_broadcast_queue( &self, mut stop_tx_bcast_receiver: tokio::sync::watch::Receiver<()>, ) { let mut receiver = self.tx_broadcaster.get_broadcast_queue().await; + // Packages whose classification failed, each waiting out FAILED_CLASSIFY_RETRY_DELAY + // before its next attempt. New packages keep flowing while these wait, and pending + // retries die with the loop on shutdown rather than resurfacing after a later start. + let mut parked: Vec<(tokio::time::Instant, BroadcastPackage)> = Vec::new(); loop { let tx_bcast_logger = Arc::clone(&self.logger); + // Entries are appended with a fixed delay, so the first is always the next due. + let next_retry_at = parked.first().map(|(deadline, _)| *deadline); tokio::select! { _ = stop_tx_bcast_receiver.changed() => { log_debug!( @@ -577,35 +624,18 @@ impl ChainSource { return; } Some(next_package) = receiver.recv() => { - // Classify funding broadcasts into payment records before sending. If - // classification fails we delay the broadcast and retry, since broadcasting - // a tx we failed to record would leave it on-chain without a payment — - // while dropping the package would not keep an interactively funded tx - // off-chain (the counterparty broadcasts it regardless), only leave it - // confirming without a recorded candidate. - if let Err(e) = self.tx_broadcaster.classify_package(&next_package).await { - log_error!( - tx_bcast_logger, - "Delaying broadcast: failed to persist payment records, will retry: {:?}", - e, - ); - self.tx_broadcaster.requeue_failed_classify(next_package); - continue; + if let Err(package) = self.classify_and_broadcast(next_package).await { + let retry_at = tokio::time::Instant::now() + FAILED_CLASSIFY_RETRY_DELAY; + parked.push((retry_at, package)); } - let package = next_package.into_sorted_transactions(); - match &self.kind { - #[cfg(feature = "chain-esplora")] - ChainSourceKind::Esplora(esplora_chain_source) => { - esplora_chain_source.process_transaction_broadcast(package).await - }, - #[cfg(feature = "chain-electrum")] - ChainSourceKind::Electrum(electrum_chain_source) => { - electrum_chain_source.process_transaction_broadcast(package).await - }, - #[cfg(feature = "chain-bitcoind")] - ChainSourceKind::Bitcoind(bitcoind_chain_source) => { - bitcoind_chain_source.process_transaction_broadcast(package).await - }, + } + _ = tokio::time::sleep_until( + next_retry_at.unwrap_or_else(tokio::time::Instant::now) + ), if next_retry_at.is_some() => { + let (_, package) = parked.remove(0); + if let Err(package) = self.classify_and_broadcast(package).await { + let retry_at = tokio::time::Instant::now() + FAILED_CLASSIFY_RETRY_DELAY; + parked.push((retry_at, package)); } } } diff --git a/src/tx_broadcaster.rs b/src/tx_broadcaster.rs index 248926d45..c40592558 100644 --- a/src/tx_broadcaster.rs +++ b/src/tx_broadcaster.rs @@ -7,7 +7,6 @@ use std::ops::Deref; use std::sync::{Mutex as StdMutex, Weak}; -use std::time::Duration; use bitcoin::Transaction; use lightning::chain::chaininterface::{ @@ -21,11 +20,6 @@ use crate::Error; const BCAST_PACKAGE_QUEUE_SIZE: usize = 256; -/// How long to wait before re-classifying a package whose classification failed. Long enough to -/// give a struggling store room to recover, short against the ~minutes until the transaction -/// could confirm. -const FAILED_CLASSIFY_RETRY_DELAY: Duration = Duration::from_secs(2); - /// A package of transactions that LDK handed to the broadcaster in one `broadcast_transactions` /// call, along with each transaction's type. Queued until the background task classifies and /// broadcasts it. Built only via [`BroadcastPackage::new`] from such a call, so unrelated @@ -141,8 +135,7 @@ where /// Classifies a queued package into payment records. Returns `Err` if any classification /// fails; callers must not broadcast the package in that case, since a crash would leave the - /// transaction on-chain without a record — but must requeue it via - /// [`Self::requeue_failed_classify`] rather than drop it. + /// transaction on-chain without a record — but must retry it later rather than drop it. pub(crate) async fn classify_package(&self, package: &BroadcastPackage) -> Result<(), Error> { let wallet_opt = self.wallet.lock().expect("lock").as_ref().and_then(Weak::upgrade); if let Some(wallet) = wallet_opt { @@ -155,20 +148,6 @@ where Ok(()) } - /// Re-sends a package whose classification failed back into the queue after a delay, so a - /// transient persistence failure delays the broadcast instead of dropping the package. - /// Dropping an interactive-funding package would not even keep its transaction off-chain — - /// the counterparty broadcasts it regardless — it would only leave the transaction - /// confirming without a recorded candidate. If the queue has closed by the time the delay - /// elapses, the node is shutting down and the package is dropped with it. - pub(crate) fn requeue_failed_classify(&self, package: BroadcastPackage) { - let sender = self.queue_sender.clone(); - tokio::spawn(async move { - tokio::time::sleep(FAILED_CLASSIFY_RETRY_DELAY).await; - let _ = sender.send(package).await; - }); - } - pub(crate) fn broadcast_unclassified_transaction(&self, tx: Transaction) { self.queue_sender.try_send(BroadcastPackage::unclassified(tx)).unwrap_or_else(|e| { log_error!(self.logger, "Failed to broadcast transactions: {}", e); diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 63c8719ae..9790e4f4c 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -4337,6 +4337,88 @@ mod tests { loop_task.await.unwrap(); } + /// A package awaiting a classification retry must die when the node stops. When the retry + /// was a detached task, it outlived the broadcast loop: its re-send into the still-open + /// queue succeeded after `stop()`, so a later `start()` would classify and broadcast the + /// stale package. + #[tokio::test] + async fn failed_classification_retry_dies_at_stop() { + use lightning::chain::chaininterface::BroadcasterInterface; + + let fail_store = FailSwitchStore::new(); + let store: Arc = Arc::new(DynStoreWrapper(fail_store.clone())); + let wallet = new_test_wallet(Arc::clone(&store), false).await; + wallet.broadcaster.set_wallet(Arc::downgrade(&wallet)); + + let (stop_sender, stop_receiver) = tokio::sync::watch::channel(()); + let chain_source = Arc::clone(&wallet.chain_source); + let loop_task = tokio::spawn(async move { + chain_source.continuously_process_broadcast_queue(stop_receiver).await + }); + + let script_pubkey = wallet + .inner + .lock() + .unwrap() + .reveal_next_address(KeychainKind::External) + .address + .script_pubkey(); + let tx = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: LockTime::ZERO, + input: Vec::new(), + output: vec![TxOut { value: Amount::from_sat(10_000), script_pubkey }], + }; + let counterparty_node_id = PublicKey::from_str( + "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", + ) + .unwrap(); + + // Queue the broadcast while payment persistence is failing and wait for the loop to + // fail a classification attempt, leaving a retry pending. + fail_store.fail_writes.store(true, Ordering::Release); + wallet.broadcaster.broadcast_transactions(&[( + &tx, + LdkTransactionType::Funding { + channels: vec![(counterparty_node_id, ChannelId([7u8; 32]))], + }, + )]); + let mut failed_writes = 0; + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(100)).await; + failed_writes = fail_store.failed_writes.load(Ordering::Acquire); + if failed_writes > 0 { + break; + } + } + assert!(failed_writes > 0, "classification never attempted a payment-store write"); + + // Stop the node with the retry still pending, then bring the loop back up with + // working persistence, as a stop()/start() cycle would. + stop_sender.send(()).unwrap(); + loop_task.await.unwrap(); + fail_store.fail_writes.store(false, Ordering::Release); + + let (stop_sender, stop_receiver) = tokio::sync::watch::channel(()); + let chain_source = Arc::clone(&wallet.chain_source); + let loop_task = tokio::spawn(async move { + chain_source.continuously_process_broadcast_queue(stop_receiver).await + }); + + // Watch well past the retry delay: the package from before the stop must not be + // classified or broadcast by the restarted loop. + for _ in 0..40 { + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + wallet.payment_store.list_page(None).await.unwrap().objects.is_empty(), + "a package from before stop() resurfaced after restart" + ); + } + + stop_sender.send(()).unwrap(); + loop_task.await.unwrap(); + } + /// Barrier test, classification-first ordering: wallet sync's confirmation handling must /// wait for classification's two-store write pair. Classification is parked between its /// payment-store and pending-store writes (the torn window) and only then is the