From 093eefa9fcee8ee2bb407800cea96bdd94fff3e6 Mon Sep 17 00:00:00 2001 From: Fmt Bot Date: Sun, 29 Mar 2026 02:10:20 +0000 Subject: [PATCH 01/15] 2026-03-29 automated rustfmt nightly --- src/io/sqlite_store/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index c69e5685d1..94e8360fc4 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -684,7 +684,6 @@ impl SqliteStoreInner { #[cfg(test)] mod tests { use super::*; - use crate::io::test_utils::{ do_read_write_remove_list_persist, do_test_store, random_storage_path, }; From c442e8e8c593dfa8e3fe48d5a83d7f203f3d4ee6 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 30 Mar 2026 10:03:04 +0200 Subject: [PATCH 02/15] Refactor `do_connect_peer` to always propagate result to subscribers Extract the connection logic into `do_connect_peer_internal` and have `do_connect_peer` act as a thin wrapper that always calls `propagate_result_to_subscribers` with the result. This removes the need to manually propagate at every error site, making the code less error-prone. Co-Authored-By: HAL 9000 --- src/connection.rs | 47 +++++++++++++++-------------------------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/src/connection.rs b/src/connection.rs index 9110ed0d96..91da6459d7 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -56,6 +56,14 @@ where pub(crate) async fn do_connect_peer( &self, node_id: PublicKey, addr: SocketAddress, + ) -> Result<(), Error> { + let res = self.do_connect_peer_internal(node_id, addr).await; + self.propagate_result_to_subscribers(&node_id, res); + res + } + + async fn do_connect_peer_internal( + &self, node_id: PublicKey, addr: SocketAddress, ) -> Result<(), Error> { // First, we check if there is already an outbound connection in flight, if so, we just // await on the corresponding watch channel. The task driving the connection future will @@ -71,15 +79,14 @@ where log_info!(self.logger, "Connecting to peer: {}@{}", node_id, addr); - let res = match addr { + match addr { SocketAddress::OnionV2(old_onion_addr) => { log_error!( - self.logger, - "Failed to resolve network address {:?}: Resolution of OnionV2 addresses is currently unsupported.", - old_onion_addr - ); - self.propagate_result_to_subscribers(&node_id, Err(Error::InvalidSocketAddress)); - return Err(Error::InvalidSocketAddress); + self.logger, + "Failed to resolve network address {:?}: Resolution of OnionV2 addresses is currently unsupported.", + old_onion_addr + ); + Err(Error::InvalidSocketAddress) }, SocketAddress::OnionV3 { .. } => { let proxy_config = self.tor_proxy_config.as_ref().ok_or_else(|| { @@ -88,10 +95,6 @@ where "Failed to resolve network address {:?}: Tor usage is not configured.", addr ); - self.propagate_result_to_subscribers( - &node_id, - Err(Error::InvalidSocketAddress), - ); Error::InvalidSocketAddress })?; let proxy_addr = proxy_config @@ -104,10 +107,6 @@ where proxy_config.proxy_address, e ); - self.propagate_result_to_subscribers( - &node_id, - Err(Error::InvalidSocketAddress), - ); Error::InvalidSocketAddress })? .next() @@ -117,10 +116,6 @@ where "Failed to resolve Tor proxy network address {}", proxy_config.proxy_address ); - self.propagate_result_to_subscribers( - &node_id, - Err(Error::InvalidSocketAddress), - ); Error::InvalidSocketAddress })?; let connection_future = lightning_net_tokio::tor_connect_outbound( @@ -142,19 +137,11 @@ where addr, e ); - self.propagate_result_to_subscribers( - &node_id, - Err(Error::InvalidSocketAddress), - ); Error::InvalidSocketAddress })? .next() .ok_or_else(|| { log_error!(self.logger, "Failed to resolve network address {}", addr); - self.propagate_result_to_subscribers( - &node_id, - Err(Error::InvalidSocketAddress), - ); Error::InvalidSocketAddress })?; let connection_future = lightning_net_tokio::connect_outbound( @@ -164,11 +151,7 @@ where ); self.await_connection(connection_future, node_id, addr).await }, - }; - - self.propagate_result_to_subscribers(&node_id, res); - - res + } } async fn await_connection( From 09e7fa44076ace0ae4c9ac3f2ddabe024d7d432a Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 30 Mar 2026 16:52:43 +0200 Subject: [PATCH 03/15] Switch Python build backend from setuptools to hatchling Replace the split setuptools configuration (pyproject.toml + setup.cfg) with a unified hatchling-based setup. This adds a [build-system] section pointing to hatchling and a build hook (hatch_build.py) that marks wheels as platform-specific since we bundle native shared libraries. Hatchling includes all files in the package directory by default, which also fixes the missing *.dll glob that setup.cfg had for Windows. Bump requires-python from >=3.6 to >=3.8 as 3.6/3.7 are long EOL. Co-Authored-By: HAL 9000 --- bindings/python/hatch_build.py | 7 +++++++ bindings/python/pyproject.toml | 9 ++++++++- bindings/python/setup.cfg | 13 ------------- 3 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 bindings/python/hatch_build.py delete mode 100644 bindings/python/setup.cfg diff --git a/bindings/python/hatch_build.py b/bindings/python/hatch_build.py new file mode 100644 index 0000000000..bd5f54d244 --- /dev/null +++ b/bindings/python/hatch_build.py @@ -0,0 +1,7 @@ +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class CustomBuildHook(BuildHookInterface): + def initialize(self, version, build_data): + build_data["pure_python"] = False + build_data["infer_tag"] = True diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 18ba319c40..58cffbea84 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "ldk_node" version = "0.7.0" @@ -6,7 +10,7 @@ authors = [ ] description = "A ready-to-go Lightning node library built using LDK and BDK." readme = "README.md" -requires-python = ">=3.6" +requires-python = ">=3.8" classifiers = [ "Topic :: Software Development :: Libraries", "Topic :: Security :: Cryptography", @@ -19,3 +23,6 @@ classifiers = [ "Homepage" = "https://lightningdevkit.org/" "Github" = "https://github.com/lightningdevkit/ldk-node" "Bug Tracker" = "https://github.com/lightningdevkit/ldk-node/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/ldk_node"] diff --git a/bindings/python/setup.cfg b/bindings/python/setup.cfg deleted file mode 100644 index bd4e642165..0000000000 --- a/bindings/python/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[options] -packages = find: -package_dir = - = src -include_package_data = True - -[options.packages.find] -where = src - -[options.package_data] -ldk_node = - *.so - *.dylib From 0da793ed254ce8c00a160f07aa95df7b2ded7dab Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 30 Mar 2026 16:52:54 +0200 Subject: [PATCH 04/15] Replace `python_create_package.sh` with uv-based build and publish scripts Add `python_build_wheel.sh` which generates bindings and builds a platform-specific wheel via `uv build`, and `python_publish_package.sh` which publishes collected wheels via `uv publish`. The intended workflow is to run the build script on each target platform (Linux, macOS), collect the wheels, and then publish them in one go. Co-Authored-By: HAL 9000 --- scripts/python_build_wheel.sh | 31 +++++++++++++++++++++++++++++++ scripts/python_create_package.sh | 3 --- scripts/python_publish_package.sh | 26 ++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100755 scripts/python_build_wheel.sh delete mode 100755 scripts/python_create_package.sh create mode 100755 scripts/python_publish_package.sh diff --git a/scripts/python_build_wheel.sh b/scripts/python_build_wheel.sh new file mode 100755 index 0000000000..4bae18479c --- /dev/null +++ b/scripts/python_build_wheel.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Build a Python wheel for the current platform. +# +# This script compiles the Rust library, generates Python bindings via UniFFI, +# and builds a platform-specific wheel using uv + hatchling. +# +# Run this on each target platform (Linux, macOS) to collect wheels, then use +# scripts/python_publish_package.sh to publish them. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +cd "$REPO_ROOT" + +# Generate bindings and compile the native library +echo "Generating Python bindings..." +./scripts/uniffi_bindgen_generate_python.sh + +# Build the wheel +echo "Building wheel..." +cd bindings/python +uv build --wheel + +echo "" +echo "Wheel built successfully:" +ls -1 dist/*.whl +echo "" +echo "Collect wheels from all target platforms into dist/, then run:" +echo " ./scripts/python_publish_package.sh" diff --git a/scripts/python_create_package.sh b/scripts/python_create_package.sh deleted file mode 100755 index 0a993c9cbd..0000000000 --- a/scripts/python_create_package.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd bindings/python || exit 1 -python3 -m build diff --git a/scripts/python_publish_package.sh b/scripts/python_publish_package.sh new file mode 100755 index 0000000000..971a4eddac --- /dev/null +++ b/scripts/python_publish_package.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Publish Python wheels to PyPI (or TestPyPI). +# +# Usage: +# ./scripts/python_publish_package.sh # publish to PyPI +# ./scripts/python_publish_package.sh --index testpypi # publish to TestPyPI +# +# Before running, collect wheels from all target platforms into bindings/python/dist/. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +DIST_DIR="$REPO_ROOT/bindings/python/dist" + +if [ ! -d "$DIST_DIR" ] || [ -z "$(ls -A "$DIST_DIR"/*.whl 2>/dev/null)" ]; then + echo "Error: No wheels found in $DIST_DIR" + echo "Run ./scripts/python_build_wheel.sh on each target platform first." + exit 1 +fi + +echo "Wheels to publish:" +ls -1 "$DIST_DIR"/*.whl +echo "" + +uv publish "$@" "$DIST_DIR"/*.whl From 0a7ae2b24832d6db256a9bd9c71b12ce7ad9ca60 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 30 Mar 2026 16:53:01 +0200 Subject: [PATCH 05/15] Switch Python CI workflow to use uv Replace `actions/setup-python` with `astral-sh/setup-uv` and use `uv run` to run tests. Co-Authored-By: HAL 9000 --- .github/workflows/python.yml | 12 +++--------- bindings/python/pyproject.toml | 7 ++++++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d9bc978d14..802f7c3d4f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -17,10 +17,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' + - name: Install uv + uses: astral-sh/setup-uv@v7 - name: Generate Python bindings run: ./scripts/uniffi_bindgen_generate_python.sh @@ -28,10 +26,6 @@ jobs: - name: Start bitcoind and electrs run: docker compose up -d - - name: Install testing prerequisites - run: | - pip3 install requests - - name: Run Python unit tests env: BITCOIN_CLI_BIN: "docker exec ldk-node-bitcoin-1 bitcoin-cli" @@ -40,4 +34,4 @@ jobs: ESPLORA_ENDPOINT: "http://127.0.0.1:3002" run: | cd $LDK_NODE_PYTHON_DIR - python3 -m unittest discover -s src/ldk_node + uv run --group dev python -m unittest discover -s src/ldk_node diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 58cffbea84..b77801d457 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -9,7 +9,7 @@ authors = [ { name="Elias Rohrer", email="dev@tnull.de" }, ] description = "A ready-to-go Lightning node library built using LDK and BDK." -readme = "README.md" +readme = "../../README.md" requires-python = ">=3.8" classifiers = [ "Topic :: Software Development :: Libraries", @@ -24,5 +24,10 @@ classifiers = [ "Github" = "https://github.com/lightningdevkit/ldk-node" "Bug Tracker" = "https://github.com/lightningdevkit/ldk-node/issues" +[dependency-groups] +dev = ["requests"] + [tool.hatch.build.targets.wheel] packages = ["src/ldk_node"] + +[tool.hatch.build.hooks.custom] From 105f8354191bdef016df2ebb246446c3a3e95e4b Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 30 Mar 2026 10:07:07 +0200 Subject: [PATCH 06/15] Replace `to_socket_addrs()` with `tokio::net::lookup_host` Replace the synchronous, blocking `std::net::ToSocketAddrs::to_socket_addrs()` calls with async `tokio::net::lookup_host` to avoid blocking the tokio runtime during DNS resolution. Additionally, instead of only using the first resolved address, we now iterate over all resolved addresses and try connecting to each in sequence until one succeeds. This improves connectivity for hostnames that resolve to multiple addresses (e.g., dual-stack IPv4/IPv6). Co-Authored-By: HAL 9000 --- Cargo.toml | 2 +- src/connection.rs | 137 +++++++++++++++++++++++++++++++--------------- src/lib.rs | 36 ++++++------ 3 files changed, 112 insertions(+), 63 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5399416777..8a85c65740 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ bip21 = { version = "0.5", features = ["std"], default-features = false } base64 = { version = "0.22.1", default-features = false, features = ["std"] } getrandom = { version = "0.3", default-features = false } chrono = { version = "0.4", default-features = false, features = ["clock"] } -tokio = { version = "1.37", default-features = false, features = [ "rt-multi-thread", "time", "sync", "macros" ] } +tokio = { version = "1.37", default-features = false, features = [ "rt-multi-thread", "time", "sync", "macros", "net" ] } esplora-client = { version = "0.12", default-features = false, features = ["tokio", "async-https-rustls"] } electrum-client = { version = "0.24.0", default-features = false, features = ["proxy", "use-rustls-ring"] } libc = "0.2" diff --git a/src/connection.rs b/src/connection.rs index 91da6459d7..a1d24e36da 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -6,7 +6,6 @@ // accordance with one or both of these licenses. use std::collections::hash_map::{self, HashMap}; -use std::net::ToSocketAddrs; use std::ops::Deref; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -15,7 +14,7 @@ use bitcoin::secp256k1::PublicKey; use lightning::ln::msgs::SocketAddress; use crate::config::TorConfig; -use crate::logger::{log_error, log_info, LdkLogger}; +use crate::logger::{log_debug, log_error, log_info, LdkLogger}; use crate::types::{KeysManager, PeerManager}; use crate::Error; @@ -97,39 +96,64 @@ where ); Error::InvalidSocketAddress })?; - let proxy_addr = proxy_config - .proxy_address - .to_socket_addrs() - .map_err(|e| { - log_error!( - self.logger, - "Failed to resolve Tor proxy network address {}: {}", - proxy_config.proxy_address, - e - ); - Error::InvalidSocketAddress - })? - .next() - .ok_or_else(|| { - log_error!( - self.logger, - "Failed to resolve Tor proxy network address {}", - proxy_config.proxy_address - ); - Error::InvalidSocketAddress - })?; - let connection_future = lightning_net_tokio::tor_connect_outbound( - Arc::clone(&self.peer_manager), - node_id, - addr.clone(), - proxy_addr, - Arc::clone(&self.keys_manager), - ); - self.await_connection(connection_future, node_id, addr).await + let resolved_addrs: Vec<_> = + tokio::net::lookup_host(proxy_config.proxy_address.to_string()) + .await + .map_err(|e| { + log_error!( + self.logger, + "Failed to resolve Tor proxy network address {}: {}", + proxy_config.proxy_address, + e + ); + Error::InvalidSocketAddress + })? + .collect(); + + if resolved_addrs.is_empty() { + log_error!( + self.logger, + "Failed to resolve Tor proxy network address {}", + proxy_config.proxy_address + ); + return Err(Error::InvalidSocketAddress); + } + + let mut res = Err(Error::ConnectionFailed); + let mut had_failures = false; + for proxy_addr in resolved_addrs { + let connection_future = lightning_net_tokio::tor_connect_outbound( + Arc::clone(&self.peer_manager), + node_id, + addr.clone(), + proxy_addr, + Arc::clone(&self.keys_manager), + ); + res = self.await_connection(connection_future, node_id, addr.clone()).await; + if res.is_ok() { + if had_failures { + log_info!( + self.logger, + "Successfully connected to peer {}@{} via resolved proxy address {} after previous attempts failed.", + node_id, addr, proxy_addr + ); + } + break; + } + had_failures = true; + log_debug!( + self.logger, + "Failed to connect to peer {}@{} via resolved proxy address {}.", + node_id, + addr, + proxy_addr + ); + } + res }, _ => { - let socket_addr = addr - .to_socket_addrs() + let resolved_addrs: Vec<_> = tokio::net::lookup_host(addr.to_string()) + .await .map_err(|e| { log_error!( self.logger, @@ -139,17 +163,42 @@ where ); Error::InvalidSocketAddress })? - .next() - .ok_or_else(|| { - log_error!(self.logger, "Failed to resolve network address {}", addr); - Error::InvalidSocketAddress - })?; - let connection_future = lightning_net_tokio::connect_outbound( - Arc::clone(&self.peer_manager), - node_id, - socket_addr, - ); - self.await_connection(connection_future, node_id, addr).await + .collect(); + + if resolved_addrs.is_empty() { + log_error!(self.logger, "Failed to resolve network address {}", addr); + return Err(Error::InvalidSocketAddress); + } + + let mut res = Err(Error::ConnectionFailed); + let mut had_failures = false; + for socket_addr in resolved_addrs { + let connection_future = lightning_net_tokio::connect_outbound( + Arc::clone(&self.peer_manager), + node_id, + socket_addr, + ); + res = self.await_connection(connection_future, node_id, addr.clone()).await; + if res.is_ok() { + if had_failures { + log_info!( + self.logger, + "Successfully connected to peer {}@{} via resolved address {} after previous attempts failed.", + node_id, addr, socket_addr + ); + } + break; + } + had_failures = true; + log_debug!( + self.logger, + "Failed to connect to peer {}@{} via resolved address {}.", + node_id, + addr, + socket_addr + ); + } + res }, } } diff --git a/src/lib.rs b/src/lib.rs index 2e02e996c0..2ac4697e81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,7 +108,6 @@ mod types; mod wallet; use std::default::Default; -use std::net::ToSocketAddrs; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; #[cfg(cycle_tests)] @@ -361,28 +360,29 @@ impl Node { let peer_manager_connection_handler = Arc::clone(&self.peer_manager); let listening_logger = Arc::clone(&self.logger); - let mut bind_addrs = Vec::with_capacity(listening_addresses.len()); - - for listening_addr in listening_addresses { - let resolved_address = listening_addr.to_socket_addrs().map_err(|e| { - log_error!( - self.logger, - "Unable to resolve listening address: {:?}. Error details: {}", - listening_addr, - e, - ); - Error::InvalidSocketAddress - })?; - - bind_addrs.extend(resolved_address); - } - let logger = Arc::clone(&listening_logger); + let listening_addrs = listening_addresses.clone(); let listeners = self.runtime.block_on(async move { + let mut bind_addrs = Vec::with_capacity(listening_addrs.len()); + + for listening_addr in &listening_addrs { + let resolved = + tokio::net::lookup_host(listening_addr.to_string()).await.map_err(|e| { + log_error!( + logger, + "Unable to resolve listening address: {:?}. Error details: {}", + listening_addr, + e, + ); + Error::InvalidSocketAddress + })?; + bind_addrs.extend(resolved); + } + let mut listeners = Vec::new(); // Try to bind to all addresses - for addr in &*bind_addrs { + for addr in &bind_addrs { match tokio::net::TcpListener::bind(addr).await { Ok(listener) => { log_trace!(logger, "Listener bound to {}", addr); From a5d91b42e03f0fbb3cee11f566a4de035902a787 Mon Sep 17 00:00:00 2001 From: benthecarman Date: Wed, 1 Apr 2026 11:53:03 -0500 Subject: [PATCH 07/15] Harden LNURL-auth request handling Enforce HTTPS for non-localhost URLs per LNURL spec and disable redirect following since the auth flow is a single GET request. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/lnurl_auth.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lnurl_auth.rs b/src/lnurl_auth.rs index 1a0def47c5..1f95b77b17 100644 --- a/src/lnurl_auth.rs +++ b/src/lnurl_auth.rs @@ -96,6 +96,13 @@ impl LnurlAuth { let domain = url.base_url(); + // Enforce HTTPS for non-localhost URLs per LNURL spec. + let is_localhost = domain == "localhost" || domain == "127.0.0.1" || domain == "[::1]"; + if url.scheme() != "https" && !is_localhost { + log_error!(self.logger, "LNURL-auth URL must use HTTPS for non-localhost domains"); + return Err(Error::InvalidLnurl); + } + // get query parameters for k1 and tag let query_params: std::collections::HashMap<_, _> = url.query_pairs().collect(); @@ -135,7 +142,7 @@ impl LnurlAuth { let auth_url = format!("{lnurl_auth_url}&sig={signature}&key={linking_public_key}"); log_debug!(self.logger, "Submitting LNURL-auth response"); - let request = bitreq::get(&auth_url); + let request = bitreq::get(&auth_url).with_max_redirects(0); let auth_response = self.client.send_async(request).await.map_err(|e| { log_error!(self.logger, "Failed to submit LNURL-auth response: {e}"); Error::LnurlAuthFailed From 38b612215d599e1e8fb83542a79a8a8579b5b444 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 2 Apr 2026 13:20:39 +0200 Subject: [PATCH 08/15] Re-pin `idna_adapter` to for MSRV builds .. we previously dropped the pin when moving MSRV to 1.85, but it seems that is not sufficient anymore.. --- .github/workflows/rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1ccade4441..188bee1663 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -41,6 +41,10 @@ jobs: - name: Check formatting on Rust ${{ matrix.toolchain }} if: matrix.check-fmt run: rustup component add rustfmt && cargo fmt --all -- --check + - name: Pin packages to allow for MSRV + if: matrix.msrv + run: | + cargo update -p idna_adapter --precise "1.2.0" --verbose # idna_adapter 1.2.1 uses ICU4X 2.2.0, requiring 1.86 and newer - name: Set RUSTFLAGS to deny warnings if: "matrix.toolchain == 'stable'" run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" From 818cccddec27a4905dbd0e419bd3bde4cd583e13 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Thu, 2 Apr 2026 09:35:35 +0200 Subject: [PATCH 09/15] Run rate limiter garbage collection before inserting new user Move the GC pass from after insertion to before, so that stale entries are reclaimed before allocating a new bucket. This avoids unnecessary growth of the user map between GC cycles. AI tools were used in preparing this commit. --- src/payment/asynchronous/rate_limiter.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/payment/asynchronous/rate_limiter.rs b/src/payment/asynchronous/rate_limiter.rs index 671b1dc72a..bf12508927 100644 --- a/src/payment/asynchronous/rate_limiter.rs +++ b/src/payment/asynchronous/rate_limiter.rs @@ -23,6 +23,8 @@ pub(crate) struct RateLimiter { max_idle: Duration, } +const MAX_USERS: usize = 10_000; + struct Bucket { tokens: u32, last_refill: Instant, @@ -36,10 +38,19 @@ impl RateLimiter { pub(crate) fn allow(&mut self, user_id: &[u8]) -> bool { let now = Instant::now(); - let entry = self.users.entry(user_id.to_vec()); - let is_new_user = matches!(entry, std::collections::hash_map::Entry::Vacant(_)); + let is_new_user = !self.users.contains_key(user_id); + + if is_new_user { + self.garbage_collect(self.max_idle); + if self.users.len() >= MAX_USERS { + return false; + } + } - let bucket = entry.or_insert(Bucket { tokens: self.capacity, last_refill: now }); + let bucket = self + .users + .entry(user_id.to_vec()) + .or_insert(Bucket { tokens: self.capacity, last_refill: now }); let elapsed = now.duration_since(bucket.last_refill); let tokens_to_add = (elapsed.as_secs_f64() / self.refill_interval.as_secs_f64()) as u32; @@ -56,11 +67,6 @@ impl RateLimiter { false }; - // Each time a new user is added, we take the opportunity to clean up old rate limits. - if is_new_user { - self.garbage_collect(self.max_idle); - } - allow } From 856c768b247ff16cc1187d7671e7baee7c0eda77 Mon Sep 17 00:00:00 2001 From: Yeji Han Date: Sat, 4 Apr 2026 15:48:30 +0900 Subject: [PATCH 10/15] feat(cbf): add BIP 157 compact block filter chain source Squashed base CBF commits (rebased onto upstream/main): - Add optional fee source from esplora/electrum - Add BIP 157 compact block filter chain source - Add CBF integration tests and documentation - Fix CBF chain source build errors and UniFFI bindings - Remove last_synced_height from cbf --- .github/workflows/python.yml | 12 +- .github/workflows/rust.yml | 10 +- Cargo.toml | 3 +- README.md | 3 +- bindings/ldk_node.udl | 5 + bindings/python/hatch_build.py | 7 - bindings/python/pyproject.toml | 16 +- bindings/python/setup.cfg | 13 + scripts/python_build_wheel.sh | 31 - scripts/python_create_package.sh | 3 + scripts/python_publish_package.sh | 26 - src/builder.rs | 98 +- src/chain/cbf.rs | 1211 ++++++++++++++++++++++ src/chain/mod.rs | 94 +- src/config.rs | 47 + src/connection.rs | 182 ++-- src/ffi/types.rs | 4 +- src/io/sqlite_store/mod.rs | 1 + src/lib.rs | 37 +- src/lnurl_auth.rs | 9 +- src/payment/asynchronous/rate_limiter.rs | 22 +- src/wallet/mod.rs | 34 +- tests/common/mod.rs | 104 +- tests/integration_tests_rust.rs | 344 +++++- 24 files changed, 2061 insertions(+), 255 deletions(-) delete mode 100644 bindings/python/hatch_build.py create mode 100644 bindings/python/setup.cfg delete mode 100755 scripts/python_build_wheel.sh create mode 100755 scripts/python_create_package.sh delete mode 100755 scripts/python_publish_package.sh create mode 100644 src/chain/cbf.rs diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 802f7c3d4f..d9bc978d14 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -17,8 +17,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v7 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' - name: Generate Python bindings run: ./scripts/uniffi_bindgen_generate_python.sh @@ -26,6 +28,10 @@ jobs: - name: Start bitcoind and electrs run: docker compose up -d + - name: Install testing prerequisites + run: | + pip3 install requests + - name: Run Python unit tests env: BITCOIN_CLI_BIN: "docker exec ldk-node-bitcoin-1 bitcoin-cli" @@ -34,4 +40,4 @@ jobs: ESPLORA_ENDPOINT: "http://127.0.0.1:3002" run: | cd $LDK_NODE_PYTHON_DIR - uv run --group dev python -m unittest discover -s src/ldk_node + python3 -m unittest discover -s src/ldk_node diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 188bee1663..fcda2c83e1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -41,10 +41,6 @@ jobs: - name: Check formatting on Rust ${{ matrix.toolchain }} if: matrix.check-fmt run: rustup component add rustfmt && cargo fmt --all -- --check - - name: Pin packages to allow for MSRV - if: matrix.msrv - run: | - cargo update -p idna_adapter --precise "1.2.0" --verbose # idna_adapter 1.2.1 uses ICU4X 2.2.0, requiring 1.86 and newer - name: Set RUSTFLAGS to deny warnings if: "matrix.toolchain == 'stable'" run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" @@ -84,7 +80,11 @@ jobs: - name: Test on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest'" run: | - RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test + RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test -- --skip cbf + - name: Test CBF on Rust ${{ matrix.toolchain }} + if: "matrix.platform != 'windows-latest'" + run: | + RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test cbf -- --test-threads=1 - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" run: | diff --git a/Cargo.toml b/Cargo.toml index 8a85c65740..8182c4f8b4 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ lightning-macros = { git = "https://github.com/lightningdevkit/rust-lightning", bdk_chain = { version = "0.23.0", default-features = false, features = ["std"] } bdk_esplora = { version = "0.22.0", default-features = false, features = ["async-https-rustls", "tokio"]} bdk_electrum = { version = "0.23.0", default-features = false, features = ["use-rustls-ring"]} +bip157 = { version = "0.4.2", default-features = false } bdk_wallet = { version = "2.3.0", default-features = false, features = ["std", "keys-bip39"]} bitreq = { version = "0.3", default-features = false, features = ["async-https", "json-using-serde"] } @@ -66,7 +67,7 @@ bip21 = { version = "0.5", features = ["std"], default-features = false } base64 = { version = "0.22.1", default-features = false, features = ["std"] } getrandom = { version = "0.3", default-features = false } chrono = { version = "0.4", default-features = false, features = ["clock"] } -tokio = { version = "1.37", default-features = false, features = [ "rt-multi-thread", "time", "sync", "macros", "net" ] } +tokio = { version = "1.37", default-features = false, features = [ "rt-multi-thread", "time", "sync", "macros" ] } esplora-client = { version = "0.12", default-features = false, features = ["tokio", "async-https-rustls"] } electrum-client = { version = "0.24.0", default-features = false, features = ["proxy", "use-rustls-ring"] } libc = "0.2" diff --git a/README.md b/README.md index 0068b6e07a..32417242b1 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ fn main() { LDK Node currently comes with a decidedly opinionated set of design choices: - On-chain data is handled by the integrated [BDK][bdk] wallet. -- Chain data may currently be sourced from the Bitcoin Core RPC interface, or from an [Electrum][electrum] or [Esplora][esplora] server. +- Chain data may currently be sourced from the Bitcoin Core RPC interface, from an [Electrum][electrum] or [Esplora][esplora] server, or via [compact block filters (BIP 157)][bip157]. - Wallet and channel state may be persisted to an [SQLite][sqlite] database, to file system, or to a custom back-end to be implemented by the user. - Gossip data may be sourced via Lightning's peer-to-peer network or the [Rapid Gossip Sync](https://docs.rs/lightning-rapid-gossip-sync/*/lightning_rapid_gossip_sync/) protocol. - Entropy for the Lightning and on-chain wallets may be sourced from raw bytes or a [BIP39](https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki) mnemonic. In addition, LDK Node offers the means to generate and persist the entropy bytes to disk. @@ -80,6 +80,7 @@ The Minimum Supported Rust Version (MSRV) is currently 1.85.0. [bdk]: https://bitcoindevkit.org/ [electrum]: https://github.com/spesmilo/electrum-protocol [esplora]: https://github.com/Blockstream/esplora +[bip157]: https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki [sqlite]: https://sqlite.org/ [rust]: https://www.rust-lang.org/ [swift]: https://www.swift.org/ diff --git a/bindings/ldk_node.udl b/bindings/ldk_node.udl index 0149936905..31d3d6a4cd 100644 --- a/bindings/ldk_node.udl +++ b/bindings/ldk_node.udl @@ -9,6 +9,8 @@ typedef dictionary EsploraSyncConfig; typedef dictionary ElectrumSyncConfig; +typedef dictionary CbfSyncConfig; + typedef dictionary TorConfig; typedef interface NodeEntropy; @@ -38,6 +40,7 @@ interface Builder { constructor(Config config); void set_chain_source_esplora(string server_url, EsploraSyncConfig? config); void set_chain_source_electrum(string server_url, ElectrumSyncConfig? config); + void set_chain_source_cbf(sequence peers, CbfSyncConfig? sync_config, FeeSourceConfig? fee_source_config); void set_chain_source_bitcoind_rpc(string rpc_host, u16 rpc_port, string rpc_user, string rpc_password); void set_chain_source_bitcoind_rest(string rest_host, u16 rest_port, string rpc_host, u16 rpc_port, string rpc_user, string rpc_password); void set_gossip_source_p2p(); @@ -354,6 +357,8 @@ enum Currency { typedef enum AsyncPaymentsRole; +typedef enum FeeSourceConfig; + [Custom] typedef string Txid; diff --git a/bindings/python/hatch_build.py b/bindings/python/hatch_build.py deleted file mode 100644 index bd5f54d244..0000000000 --- a/bindings/python/hatch_build.py +++ /dev/null @@ -1,7 +0,0 @@ -from hatchling.builders.hooks.plugin.interface import BuildHookInterface - - -class CustomBuildHook(BuildHookInterface): - def initialize(self, version, build_data): - build_data["pure_python"] = False - build_data["infer_tag"] = True diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index b77801d457..18ba319c40 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,7 +1,3 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - [project] name = "ldk_node" version = "0.7.0" @@ -9,8 +5,8 @@ authors = [ { name="Elias Rohrer", email="dev@tnull.de" }, ] description = "A ready-to-go Lightning node library built using LDK and BDK." -readme = "../../README.md" -requires-python = ">=3.8" +readme = "README.md" +requires-python = ">=3.6" classifiers = [ "Topic :: Software Development :: Libraries", "Topic :: Security :: Cryptography", @@ -23,11 +19,3 @@ classifiers = [ "Homepage" = "https://lightningdevkit.org/" "Github" = "https://github.com/lightningdevkit/ldk-node" "Bug Tracker" = "https://github.com/lightningdevkit/ldk-node/issues" - -[dependency-groups] -dev = ["requests"] - -[tool.hatch.build.targets.wheel] -packages = ["src/ldk_node"] - -[tool.hatch.build.hooks.custom] diff --git a/bindings/python/setup.cfg b/bindings/python/setup.cfg new file mode 100644 index 0000000000..bd4e642165 --- /dev/null +++ b/bindings/python/setup.cfg @@ -0,0 +1,13 @@ +[options] +packages = find: +package_dir = + = src +include_package_data = True + +[options.packages.find] +where = src + +[options.package_data] +ldk_node = + *.so + *.dylib diff --git a/scripts/python_build_wheel.sh b/scripts/python_build_wheel.sh deleted file mode 100755 index 4bae18479c..0000000000 --- a/scripts/python_build_wheel.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Build a Python wheel for the current platform. -# -# This script compiles the Rust library, generates Python bindings via UniFFI, -# and builds a platform-specific wheel using uv + hatchling. -# -# Run this on each target platform (Linux, macOS) to collect wheels, then use -# scripts/python_publish_package.sh to publish them. - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -cd "$REPO_ROOT" - -# Generate bindings and compile the native library -echo "Generating Python bindings..." -./scripts/uniffi_bindgen_generate_python.sh - -# Build the wheel -echo "Building wheel..." -cd bindings/python -uv build --wheel - -echo "" -echo "Wheel built successfully:" -ls -1 dist/*.whl -echo "" -echo "Collect wheels from all target platforms into dist/, then run:" -echo " ./scripts/python_publish_package.sh" diff --git a/scripts/python_create_package.sh b/scripts/python_create_package.sh new file mode 100755 index 0000000000..0a993c9cbd --- /dev/null +++ b/scripts/python_create_package.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd bindings/python || exit 1 +python3 -m build diff --git a/scripts/python_publish_package.sh b/scripts/python_publish_package.sh deleted file mode 100755 index 971a4eddac..0000000000 --- a/scripts/python_publish_package.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Publish Python wheels to PyPI (or TestPyPI). -# -# Usage: -# ./scripts/python_publish_package.sh # publish to PyPI -# ./scripts/python_publish_package.sh --index testpypi # publish to TestPyPI -# -# Before running, collect wheels from all target platforms into bindings/python/dist/. - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DIST_DIR="$REPO_ROOT/bindings/python/dist" - -if [ ! -d "$DIST_DIR" ] || [ -z "$(ls -A "$DIST_DIR"/*.whl 2>/dev/null)" ]; then - echo "Error: No wheels found in $DIST_DIR" - echo "Run ./scripts/python_build_wheel.sh on each target platform first." - exit 1 -fi - -echo "Wheels to publish:" -ls -1 "$DIST_DIR"/*.whl -echo "" - -uv publish "$@" "$DIST_DIR"/*.whl diff --git a/src/builder.rs b/src/builder.rs index cd8cc184fd..5a388cd82b 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -42,11 +42,11 @@ use lightning::util::sweep::OutputSweeper; use lightning_persister::fs_store::v1::FilesystemStore; use vss_client::headers::VssHeaderProvider; -use crate::chain::ChainSource; +use crate::chain::{ChainSource, FeeSourceConfig}; use crate::config::{ default_user_config, may_announce_channel, AnnounceError, AsyncPaymentsRole, - BitcoindRestClientConfig, Config, ElectrumSyncConfig, EsploraSyncConfig, TorConfig, - DEFAULT_ESPLORA_SERVER_URL, DEFAULT_LOG_FILENAME, DEFAULT_LOG_LEVEL, + BitcoindRestClientConfig, CbfSyncConfig, Config, ElectrumSyncConfig, EsploraSyncConfig, + TorConfig, DEFAULT_ESPLORA_SERVER_URL, DEFAULT_LOG_FILENAME, DEFAULT_LOG_LEVEL, }; use crate::connection::ConnectionManager; use crate::entropy::NodeEntropy; @@ -105,6 +105,11 @@ enum ChainDataSourceConfig { rpc_password: String, rest_client_config: Option, }, + Cbf { + peers: Vec, + sync_config: Option, + fee_source_config: Option, + }, } #[derive(Debug, Clone)] @@ -193,6 +198,8 @@ pub enum BuildError { NetworkMismatch, /// The role of the node in an asynchronous payments context is not compatible with the current configuration. AsyncPaymentsConfigMismatch, + /// We failed to setup the chain source. + ChainSourceSetupFailed, } impl fmt::Display for BuildError { @@ -226,6 +233,7 @@ impl fmt::Display for BuildError { "The async payments role is not compatible with the current configuration." ) }, + Self::ChainSourceSetupFailed => write!(f, "Failed to setup chain source."), } } } @@ -365,6 +373,28 @@ impl NodeBuilder { self } + /// Configures the [`Node`] instance to source its chain data via BIP 157 compact block + /// filters. + /// + /// `peers` is an optional list of peer addresses to connect to for sourcing compact block + /// filters. If empty, the node will discover peers via DNS seeds. + /// + /// If no `sync_config` is given, default values are used. See [`CbfSyncConfig`] for more + /// information. + /// + /// Note: fee rate estimation with this chain source uses block-level averages (total fees + /// divided by block weight) rather than per-transaction fee rates. This can underestimate + /// next-block inclusion rates during periods of high mempool congestion. Percentile-based + /// target selection partially mitigates this. + pub fn set_chain_source_cbf( + &mut self, peers: Vec, sync_config: Option, + fee_source_config: Option, + ) -> &mut Self { + self.chain_data_source_config = + Some(ChainDataSourceConfig::Cbf { peers, sync_config, fee_source_config }); + self + } + /// Configures the [`Node`] instance to connect to a Bitcoin Core node via RPC. /// /// This method establishes an RPC connection that enables all essential chain operations including @@ -892,6 +922,26 @@ impl ArcedNodeBuilder { self.inner.write().unwrap().set_chain_source_electrum(server_url, sync_config); } + /// Configures the [`Node`] instance to source its chain data via BIP 157 compact block + /// filters. + /// + /// `peers` is an optional list of peer addresses to connect to for sourcing compact block + /// filters. If empty, the node will discover peers via DNS seeds. + /// + /// If no `sync_config` is given, default values are used. See [`CbfSyncConfig`] for more + /// information. + /// + /// Note: fee rate estimation with this chain source uses block-level averages (total fees + /// divided by block weight) rather than per-transaction fee rates. This can underestimate + /// next-block inclusion rates during periods of high mempool congestion. Percentile-based + /// target selection partially mitigates this. + pub fn set_chain_source_cbf( + &self, peers: Vec, sync_config: Option, + fee_source_config: Option, + ) { + self.inner.write().unwrap().set_chain_source_cbf(peers, sync_config, fee_source_config); + } + /// Configures the [`Node`] instance to connect to a Bitcoin Core node via RPC. /// /// This method establishes an RPC connection that enables all essential chain operations including @@ -1364,6 +1414,25 @@ fn build_with_store_internal( }), }, + Some(ChainDataSourceConfig::Cbf { peers, sync_config, fee_source_config }) => { + let sync_config = sync_config.clone().unwrap_or(CbfSyncConfig::default()); + ChainSource::new_cbf( + peers.clone(), + sync_config, + fee_source_config.clone(), + Arc::clone(&fee_estimator), + Arc::clone(&tx_broadcaster), + Arc::clone(&kv_store), + Arc::clone(&config), + Arc::clone(&logger), + Arc::clone(&node_metrics), + ) + .map_err(|e| { + log_error!(logger, "Failed to initialize CBF chain source: {}", e); + BuildError::ChainSourceSetupFailed + })? + }, + None => { // Default to Esplora client. let server_url = DEFAULT_ESPLORA_SERVER_URL.to_string(); @@ -2079,6 +2148,9 @@ pub(crate) fn sanitize_alias(alias_str: &str) -> Result { #[cfg(test)] mod tests { + #[cfg(feature = "uniffi")] + use crate::config::CbfSyncConfig; + use super::{sanitize_alias, BuildError, NodeAlias}; #[test] @@ -2116,4 +2188,24 @@ mod tests { let node = sanitize_alias(alias); assert_eq!(node.err().unwrap(), BuildError::InvalidNodeAlias); } + + #[cfg(feature = "uniffi")] + #[test] + fn arced_builder_can_set_cbf_chain_source() { + let builder = super::ArcedNodeBuilder::new(); + let sync_config = CbfSyncConfig::default(); + + let peers = vec!["127.0.0.1:8333".to_string()]; + builder.set_chain_source_cbf(peers.clone(), Some(sync_config.clone()), None); + + let guard = builder.inner.read().unwrap(); + assert!(matches!( + guard.chain_data_source_config.as_ref(), + Some(super::ChainDataSourceConfig::Cbf { + peers: p, + sync_config: Some(config), + fee_source_config: None, + }) if config == &sync_config && p == &peers + )); + } } diff --git a/src/chain/cbf.rs b/src/chain/cbf.rs new file mode 100644 index 0000000000..0c76401967 --- /dev/null +++ b/src/chain/cbf.rs @@ -0,0 +1,1211 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +use std::collections::{BTreeMap, HashMap}; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use bdk_chain::{BlockId, ConfirmationBlockTime, TxUpdate}; +use bdk_wallet::Update; +use bip157::chain::BlockHeaderChanges; +use bip157::{ + BlockHash, Builder, Client, Event, Info, Requester, SyncUpdate, TrustedPeer, Warning, +}; +use bitcoin::constants::SUBSIDY_HALVING_INTERVAL; +use bitcoin::{Amount, FeeRate, Network, Script, ScriptBuf, Transaction, Txid}; +use electrum_client::ElectrumApi; +use lightning::chain::{Confirm, WatchedOutput}; +use lightning::util::ser::Writeable; +use tokio::sync::{mpsc, oneshot}; + +use super::{FeeSourceConfig, WalletSyncStatus}; +use crate::config::{CbfSyncConfig, Config, BDK_CLIENT_STOP_GAP}; +use crate::error::Error; +use crate::fee_estimator::{ + apply_post_estimation_adjustments, get_all_conf_targets, get_num_block_defaults_for_target, + OnchainFeeEstimator, +}; +use crate::io::utils::write_node_metrics; +use crate::logger::{log_bytes, log_debug, log_error, log_info, log_trace, LdkLogger, Logger}; +use crate::runtime::Runtime; +use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; +use crate::NodeMetrics; + +/// Minimum fee rate: 1 sat/vB = 250 sat/kWU. Used as a floor for computed fee rates. +const MIN_FEERATE_SAT_PER_KWU: u64 = 250; + +/// Number of recent blocks to look back for per-target fee rate estimation. +const FEE_RATE_LOOKBACK_BLOCKS: usize = 6; + +/// Number of blocks to walk back from a component's persisted best block height +/// for reorg safety when computing the incremental scan skip height. +/// Matches bdk-kyoto's `IMPOSSIBLE_REORG_DEPTH`. +const REORG_SAFETY_BLOCKS: u32 = 7; + +/// The fee estimation back-end used by the CBF chain source. +enum FeeSource { + /// Derive fee rates from the coinbase reward of recent blocks. + /// + /// Provides a per-target rate using percentile selection across multiple blocks. + /// Less accurate than a mempool-aware source but requires no extra connectivity. + Cbf, + /// Delegate fee estimation to an Esplora HTTP server. + Esplora { client: esplora_client::AsyncClient }, + /// Delegate fee estimation to an Electrum server. + /// + /// A fresh connection is opened for each estimation cycle because `ElectrumClient` + /// is not `Sync`. + Electrum { server_url: String }, +} + +pub(super) struct CbfChainSource { + /// Peer addresses for sourcing compact block filters via P2P. + peers: Vec, + /// User-provided sync configuration (timeouts, background sync intervals). + pub(super) sync_config: CbfSyncConfig, + /// Fee estimation back-end. + fee_source: FeeSource, + /// Tracks whether the bip157 node is running and holds the command handle. + cbf_runtime_status: Mutex, + /// Latest chain tip hash, updated by the background event processing task. + latest_tip: Arc>>, + /// Scripts to match against compact block filters during a scan. + watched_scripts: Arc>>, + /// Block (height, hash) pairs where filters matched watched scripts. + matched_block_hashes: Arc>>, + /// One-shot channel sender to signal filter scan completion. + sync_completion_tx: Arc>>>, + /// Filters at or below this height are skipped during incremental scans. + filter_skip_height: Arc, + /// Serializes concurrent filter scans (on-chain and lightning). + scan_lock: tokio::sync::Mutex<()>, + /// Scripts registered by LDK's Filter trait for lightning channel monitoring. + registered_scripts: Mutex>, + /// Deduplicates concurrent on-chain wallet sync requests. + onchain_wallet_sync_status: Mutex, + /// Deduplicates concurrent lightning wallet sync requests. + lightning_wallet_sync_status: Mutex, + /// Shared fee rate estimator, updated by this chain source. + fee_estimator: Arc, + /// Persistent key-value store for node metrics. + kv_store: Arc, + /// Node configuration (network, storage path, etc.). + config: Arc, + /// Logger instance. + logger: Arc, + /// Shared node metrics (sync timestamps, etc.). + node_metrics: Arc>, +} + +enum CbfRuntimeStatus { + Started { requester: Requester }, + Stopped, +} + +/// Shared state passed to the background event processing task. +struct CbfEventState { + latest_tip: Arc>>, + watched_scripts: Arc>>, + matched_block_hashes: Arc>>, + sync_completion_tx: Arc>>>, + filter_skip_height: Arc, +} + +impl CbfChainSource { + pub(crate) fn new( + peers: Vec, sync_config: CbfSyncConfig, fee_source_config: Option, + fee_estimator: Arc, kv_store: Arc, config: Arc, + logger: Arc, node_metrics: Arc>, + ) -> Result { + let fee_source = match fee_source_config { + Some(FeeSourceConfig::Esplora(server_url)) => { + let timeout = sync_config.timeouts_config.per_request_timeout_secs; + let mut builder = esplora_client::Builder::new(&server_url); + builder = builder.timeout(timeout as u64); + let client = builder.build_async().map_err(|e| { + log_error!(logger, "Failed to build esplora client: {}", e); + Error::ConnectionFailed + })?; + FeeSource::Esplora { client } + }, + Some(FeeSourceConfig::Electrum(server_url)) => FeeSource::Electrum { server_url }, + None => FeeSource::Cbf, + }; + + let cbf_runtime_status = Mutex::new(CbfRuntimeStatus::Stopped); + let latest_tip = Arc::new(Mutex::new(None)); + let watched_scripts = Arc::new(RwLock::new(Vec::new())); + let matched_block_hashes = Arc::new(Mutex::new(Vec::new())); + let sync_completion_tx = Arc::new(Mutex::new(None)); + let filter_skip_height = Arc::new(AtomicU32::new(0)); + let registered_scripts = Mutex::new(Vec::new()); + let scan_lock = tokio::sync::Mutex::new(()); + let onchain_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); + let lightning_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); + Ok(Self { + peers, + sync_config, + fee_source, + cbf_runtime_status, + latest_tip, + watched_scripts, + matched_block_hashes, + sync_completion_tx, + filter_skip_height, + registered_scripts, + scan_lock, + onchain_wallet_sync_status, + lightning_wallet_sync_status, + fee_estimator, + kv_store, + config, + logger, + node_metrics, + }) + } + + /// Start the bip157 node and spawn background tasks for event processing. + pub(crate) fn start(&self, runtime: Arc) { + let mut status = self.cbf_runtime_status.lock().unwrap(); + if matches!(*status, CbfRuntimeStatus::Started { .. }) { + debug_assert!(false, "We shouldn't call start if we're already started"); + return; + } + + let network = self.config.network; + + let mut builder = Builder::new(network); + + // Configure data directory under the node's storage path. + let data_dir = std::path::PathBuf::from(&self.config.storage_dir_path).join("bip157_data"); + builder = builder.data_dir(data_dir); + + // Add configured peers. + let peers: Vec = self + .peers + .iter() + .filter_map(|peer_str| { + peer_str.parse::().ok().map(TrustedPeer::from_socket_addr) + }) + .collect(); + if !peers.is_empty() { + builder = builder.add_peers(peers); + } + + // Require multiple peers to agree on filter headers before accepting them, + // as recommended by BIP 157 to mitigate malicious peer attacks. + builder = builder.required_peers(self.sync_config.required_peers); + + // Request witness data so segwit transactions include full witnesses, + // required for Lightning channel operations. + builder = builder.fetch_witness_data(); + + // Set peer response timeout from user configuration (default: 30s). + builder = + builder.response_timeout(Duration::from_secs(self.sync_config.response_timeout_secs)); + + let (node, client) = builder.build(); + + let Client { requester, info_rx, warn_rx, event_rx } = client; + + // Spawn the bip157 node in the background. + let node_logger = Arc::clone(&self.logger); + runtime.spawn_background_task(async move { + if let Err(e) = node.run().await { + log_error!(node_logger, "CBF node exited with error: {:?}", e); + } + }); + + // Spawn a task to log info messages. + let info_logger = Arc::clone(&self.logger); + runtime + .spawn_cancellable_background_task(Self::process_info_messages(info_rx, info_logger)); + + // Spawn a task to log warning messages. + let warn_logger = Arc::clone(&self.logger); + runtime + .spawn_cancellable_background_task(Self::process_warn_messages(warn_rx, warn_logger)); + + // Spawn a task to process events. + let event_state = CbfEventState { + latest_tip: Arc::clone(&self.latest_tip), + watched_scripts: Arc::clone(&self.watched_scripts), + matched_block_hashes: Arc::clone(&self.matched_block_hashes), + sync_completion_tx: Arc::clone(&self.sync_completion_tx), + filter_skip_height: Arc::clone(&self.filter_skip_height), + }; + let event_logger = Arc::clone(&self.logger); + runtime.spawn_cancellable_background_task(Self::process_events( + event_rx, + event_state, + event_logger, + )); + + log_info!(self.logger, "CBF chain source started."); + + *status = CbfRuntimeStatus::Started { requester }; + } + + /// Shut down the bip157 node and stop all background tasks. + pub(crate) fn stop(&self) { + let mut status = self.cbf_runtime_status.lock().unwrap(); + match &*status { + CbfRuntimeStatus::Started { requester } => { + let _ = requester.shutdown(); + log_info!(self.logger, "CBF chain source stopped."); + }, + CbfRuntimeStatus::Stopped => {}, + } + *status = CbfRuntimeStatus::Stopped; + } + + async fn process_info_messages(mut info_rx: mpsc::Receiver, logger: Arc) { + while let Some(info) = info_rx.recv().await { + log_debug!(logger, "CBF node info: {}", info); + } + } + + async fn process_warn_messages( + mut warn_rx: mpsc::UnboundedReceiver, logger: Arc, + ) { + while let Some(warning) = warn_rx.recv().await { + log_debug!(logger, "CBF node warning: {}", warning); + } + } + + async fn process_events( + mut event_rx: mpsc::UnboundedReceiver, state: CbfEventState, logger: Arc, + ) { + while let Some(event) = event_rx.recv().await { + match event { + Event::FiltersSynced(sync_update) => { + let tip = sync_update.tip(); + *state.latest_tip.lock().unwrap() = Some(tip.hash); + log_info!( + logger, + "CBF filters synced to tip: height={}, hash={}", + tip.height, + tip.hash, + ); + if let Some(tx) = state.sync_completion_tx.lock().unwrap().take() { + let _ = tx.send(sync_update); + } + }, + Event::Block(_) => {}, + Event::ChainUpdate(header_changes) => match header_changes { + BlockHeaderChanges::Reorganized { accepted, reorganized } => { + log_debug!( + logger, + "CBF chain reorg detected: {} blocks removed, {} blocks accepted.", + reorganized.len(), + accepted.len(), + ); + + // No height reset needed: skip heights are derived from + // BDK's checkpoint (on-chain) and LDK's best block + // (lightning), both walked back by REORG_SAFETY_BLOCKS. + }, + BlockHeaderChanges::Connected(header) => { + log_trace!(logger, "CBF block connected at height {}", header.height,); + }, + BlockHeaderChanges::ForkAdded(header) => { + log_trace!(logger, "CBF fork block observed at height {}", header.height,); + }, + }, + Event::IndexedFilter(indexed_filter) => { + let skip_height = state.filter_skip_height.load(Ordering::Acquire); + if skip_height > 0 && indexed_filter.height() <= skip_height { + continue; + } + let scripts = state.watched_scripts.read().unwrap(); + if !scripts.is_empty() && indexed_filter.contains_any(scripts.iter()) { + state + .matched_block_hashes + .lock() + .unwrap() + .push((indexed_filter.height(), indexed_filter.block_hash())); + } + log_trace!(logger, "CBF received filter at height {}", indexed_filter.height(),); + }, + } + } + } + + fn requester(&self) -> Result { + let status = self.cbf_runtime_status.lock().unwrap(); + match &*status { + CbfRuntimeStatus::Started { requester } => Ok(requester.clone()), + CbfRuntimeStatus::Stopped => { + debug_assert!( + false, + "We should have started the chain source before using the requester" + ); + Err(Error::ConnectionFailed) + }, + } + } + + /// Register a transaction script for Lightning channel monitoring. + pub(crate) fn register_tx(&self, _txid: &Txid, script_pubkey: &Script) { + self.registered_scripts.lock().unwrap().push(script_pubkey.to_owned()); + } + + /// Register a watched output script for Lightning channel monitoring. + pub(crate) fn register_output(&self, output: WatchedOutput) { + self.registered_scripts.lock().unwrap().push(output.script_pubkey.clone()); + } + + /// Run a CBF filter scan: set watched scripts, trigger a rescan, wait for + /// completion, and return the sync update along with matched block hashes. + /// + /// When `skip_before_height` is `Some(h)`, filters at or below height `h` are + /// skipped, making the scan incremental. + async fn run_filter_scan( + &self, scripts: Vec, skip_before_height: Option, + ) -> Result<(SyncUpdate, Vec<(u32, BlockHash)>), Error> { + let requester = self.requester()?; + + let _scan_guard = self.scan_lock.lock().await; + + self.filter_skip_height.store(skip_before_height.unwrap_or(0), Ordering::Release); + self.matched_block_hashes.lock().unwrap().clear(); + *self.watched_scripts.write().unwrap() = scripts; + + let (tx, rx) = oneshot::channel(); + *self.sync_completion_tx.lock().unwrap() = Some(tx); + + requester.rescan().map_err(|e| { + log_error!(self.logger, "Failed to trigger CBF rescan: {:?}", e); + Error::WalletOperationFailed + })?; + + let sync_update = rx.await.map_err(|e| { + log_error!(self.logger, "CBF sync completion channel dropped: {:?}", e); + Error::WalletOperationFailed + })?; + + self.filter_skip_height.store(0, Ordering::Release); + self.watched_scripts.write().unwrap().clear(); + let matched = std::mem::take(&mut *self.matched_block_hashes.lock().unwrap()); + + Ok((sync_update, matched)) + } + + /// Sync the on-chain wallet by scanning compact block filters for relevant transactions. + pub(crate) async fn sync_onchain_wallet( + &self, onchain_wallet: Arc, + ) -> Result<(), Error> { + let receiver_res = { + let mut status_lock = self.onchain_wallet_sync_status.lock().unwrap(); + status_lock.register_or_subscribe_pending_sync() + }; + if let Some(mut sync_receiver) = receiver_res { + log_debug!(self.logger, "On-chain wallet sync already in progress, waiting."); + return sync_receiver.recv().await.map_err(|e| { + debug_assert!(false, "Failed to receive wallet sync result: {:?}", e); + log_error!(self.logger, "Failed to receive wallet sync result: {:?}", e); + Error::WalletOperationFailed + })?; + } + + let res = async { + let requester = self.requester()?; + let now = Instant::now(); + + let scripts = onchain_wallet.get_spks_for_cbf_sync(BDK_CLIENT_STOP_GAP); + if scripts.is_empty() { + log_debug!(self.logger, "No wallet scripts to sync via CBF."); + return Ok(()); + } + + let timeout_fut = tokio::time::timeout( + Duration::from_secs( + self.sync_config.timeouts_config.onchain_wallet_sync_timeout_secs, + ), + self.sync_onchain_wallet_op(requester, &onchain_wallet, scripts), + ); + + let (tx_update, sync_update) = match timeout_fut.await { + Ok(res) => res?, + Err(e) => { + log_error!(self.logger, "Sync of on-chain wallet timed out: {}", e); + return Err(Error::WalletOperationTimeout); + }, + }; + + // Build chain checkpoint extending from the wallet's current tip. + let mut cp = onchain_wallet.latest_checkpoint(); + for (height, header) in sync_update.recent_history() { + if *height > cp.height() { + let block_id = BlockId { height: *height, hash: header.block_hash() }; + cp = cp.push(block_id).unwrap_or_else(|old| old); + } + } + let tip = sync_update.tip(); + if tip.height > cp.height() { + let tip_block_id = BlockId { height: tip.height, hash: tip.hash }; + cp = cp.push(tip_block_id).unwrap_or_else(|old| old); + } + + let update = + Update { last_active_indices: BTreeMap::new(), tx_update, chain: Some(cp) }; + + onchain_wallet.apply_update(update)?; + + log_debug!( + self.logger, + "Sync of on-chain wallet via CBF finished in {}ms.", + now.elapsed().as_millis() + ); + + update_node_metrics_timestamp( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m, t| { + m.latest_onchain_wallet_sync_timestamp = t; + }, + )?; + + Ok(()) + } + .await; + + self.onchain_wallet_sync_status.lock().unwrap().propagate_result_to_subscribers(res); + + res + } + + async fn sync_onchain_wallet_op( + &self, requester: Requester, onchain_wallet: &Wallet, scripts: Vec, + ) -> Result<(TxUpdate, SyncUpdate), Error> { + // Derive skip height from BDK's persisted checkpoint, walked back by + // REORG_SAFETY_BLOCKS for reorg safety (same approach as bdk-kyoto). + // This survives restarts since BDK persists its checkpoint chain. + // + // We include LDK-registered scripts (e.g., channel funding output + // scripts) alongside the wallet scripts. This ensures the on-chain + // wallet scan also fetches blocks containing channel funding + // transactions, whose outputs are needed by BDK's TxGraph to + // calculate fees for subsequent spends such as splice transactions. + // Without these, BDK's `calculate_fee` would fail with + // `MissingTxOut` because the parent transaction's outputs are + // unknown. This mirrors what the Bitcoind chain source does in + // `Wallet::block_connected` by inserting registered tx outputs. + let mut all_scripts = scripts; + all_scripts.extend(self.registered_scripts.lock().unwrap().iter().cloned()); + let skip_height = + onchain_wallet.latest_checkpoint().height().checked_sub(REORG_SAFETY_BLOCKS); + let (sync_update, matched) = self.run_filter_scan(all_scripts, skip_height).await?; + + log_debug!( + self.logger, + "CBF on-chain filter scan complete: {} matching blocks found.", + matched.len() + ); + + // Fetch matching blocks and include all their transactions. + // The compact block filter already matched our scripts (covering both + // created outputs and spent inputs), so we include every transaction + // from matched blocks and let BDK determine relevance. + let mut tx_update = TxUpdate::default(); + for (height, block_hash) in &matched { + let indexed_block = requester.get_block(*block_hash).await.map_err(|e| { + log_error!(self.logger, "Failed to fetch block {}: {:?}", block_hash, e); + Error::WalletOperationFailed + })?; + let block = indexed_block.block; + let block_id = BlockId { height: *height, hash: block.header.block_hash() }; + let conf_time = + ConfirmationBlockTime { block_id, confirmation_time: block.header.time as u64 }; + for tx in &block.txdata { + let txid = tx.compute_txid(); + tx_update.txs.push(Arc::new(tx.clone())); + tx_update.anchors.insert((conf_time, txid)); + } + } + + Ok((tx_update, sync_update)) + } + + /// Sync the Lightning wallet by confirming channel transactions via compact block filters. + pub(crate) async fn sync_lightning_wallet( + &self, channel_manager: Arc, chain_monitor: Arc, + output_sweeper: Arc, + ) -> Result<(), Error> { + let receiver_res = { + let mut status_lock = self.lightning_wallet_sync_status.lock().unwrap(); + status_lock.register_or_subscribe_pending_sync() + }; + if let Some(mut sync_receiver) = receiver_res { + log_debug!(self.logger, "Lightning wallet sync already in progress, waiting."); + return sync_receiver.recv().await.map_err(|e| { + debug_assert!(false, "Failed to receive wallet sync result: {:?}", e); + log_error!(self.logger, "Failed to receive wallet sync result: {:?}", e); + Error::TxSyncFailed + })?; + } + + let res = async { + let requester = self.requester()?; + let now = Instant::now(); + + let scripts: Vec = self.registered_scripts.lock().unwrap().clone(); + if scripts.is_empty() { + log_debug!(self.logger, "No registered scripts for CBF lightning sync."); + return Ok(()); + } + + let timeout_fut = tokio::time::timeout( + Duration::from_secs( + self.sync_config.timeouts_config.lightning_wallet_sync_timeout_secs, + ), + self.sync_lightning_wallet_op( + requester, + channel_manager, + chain_monitor, + output_sweeper, + scripts, + ), + ); + + match timeout_fut.await { + Ok(res) => res?, + Err(e) => { + log_error!(self.logger, "Sync of Lightning wallet timed out: {}", e); + return Err(Error::TxSyncTimeout); + }, + }; + + log_debug!( + self.logger, + "Sync of Lightning wallet via CBF finished in {}ms.", + now.elapsed().as_millis() + ); + + update_node_metrics_timestamp( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m, t| { + m.latest_lightning_wallet_sync_timestamp = t; + }, + )?; + + Ok(()) + } + .await; + + self.lightning_wallet_sync_status.lock().unwrap().propagate_result_to_subscribers(res); + + res + } + + async fn sync_lightning_wallet_op( + &self, requester: Requester, channel_manager: Arc, + chain_monitor: Arc, output_sweeper: Arc, scripts: Vec, + ) -> Result<(), Error> { + let skip_height = + channel_manager.current_best_block().height.checked_sub(REORG_SAFETY_BLOCKS); + let (sync_update, matched) = self.run_filter_scan(scripts, skip_height).await?; + + log_debug!( + self.logger, + "CBF lightning filter scan complete: {} matching blocks found.", + matched.len() + ); + + let confirmables: Vec<&(dyn Confirm + Sync + Send)> = + vec![&*channel_manager, &*chain_monitor, &*output_sweeper]; + + // Fetch matching blocks and confirm all their transactions. + // The compact block filter already matched our scripts (covering both + // created outputs and spent inputs), so we confirm every transaction + // from matched blocks and let LDK determine relevance. + for (height, block_hash) in &matched { + confirm_block_transactions( + &requester, + *block_hash, + *height, + &confirmables, + &self.logger, + ) + .await?; + } + + // Update the best block tip. + let tip = sync_update.tip(); + if let Some(tip_header) = sync_update.recent_history().get(&tip.height) { + for confirmable in &confirmables { + confirmable.best_block_updated(tip_header, tip.height); + } + } + + Ok(()) + } + + pub(crate) async fn update_fee_rate_estimates(&self) -> Result<(), Error> { + let new_fee_rate_cache = match &self.fee_source { + FeeSource::Cbf => self.fee_rate_cache_from_cbf().await?, + FeeSource::Esplora { client } => Some(self.fee_rate_cache_from_esplora(client).await?), + FeeSource::Electrum { server_url } => { + Some(self.fee_rate_cache_from_electrum(server_url).await?) + }, + }; + + let Some(new_fee_rate_cache) = new_fee_rate_cache else { + return Ok(()); + }; + + self.fee_estimator.set_fee_rate_cache(new_fee_rate_cache); + + update_node_metrics_timestamp( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m, t| { + m.latest_fee_rate_cache_update_timestamp = t; + }, + )?; + + Ok(()) + } + + /// Derive per-target fee rates from recent blocks' coinbase outputs. + /// + /// Returns `Ok(None)` when no chain tip is available yet (first startup before sync). + async fn fee_rate_cache_from_cbf( + &self, + ) -> Result>, Error> { + let requester = self.requester()?; + + let tip_hash = match *self.latest_tip.lock().unwrap() { + Some(hash) => hash, + None => { + log_debug!(self.logger, "No tip available yet for fee rate estimation, skipping."); + return Ok(None); + }, + }; + + let now = Instant::now(); + + // Fetch fee rates from the last N blocks for per-target estimation. + // We compute fee rates ourselves rather than using Requester::average_fee_rate, + // so we can sample multiple blocks and select percentiles per confirmation target. + let mut block_fee_rates: Vec = Vec::with_capacity(FEE_RATE_LOOKBACK_BLOCKS); + let mut current_hash = tip_hash; + + let timeout = Duration::from_secs( + self.sync_config.timeouts_config.fee_rate_cache_update_timeout_secs, + ); + let fetch_start = Instant::now(); + + for idx in 0..FEE_RATE_LOOKBACK_BLOCKS { + // Check if we've exceeded the overall timeout for fee estimation. + let remaining_timeout = timeout.saturating_sub(fetch_start.elapsed()); + if remaining_timeout.is_zero() { + log_error!(self.logger, "Updating fee rate estimates timed out."); + return Err(Error::FeerateEstimationUpdateTimeout); + } + + // Fetch the block via P2P. On the first iteration, a fetch failure + // likely means the cached tip is stale (initial sync or reorg), so + // we clear the tip and skip gracefully instead of returning an error. + let indexed_block = + match tokio::time::timeout(remaining_timeout, requester.get_block(current_hash)) + .await + { + Ok(Ok(indexed_block)) => indexed_block, + Ok(Err(e)) if idx == 0 => { + log_debug!( + self.logger, + "Cached CBF tip {} was unavailable during fee estimation, \ + likely due to initial sync or a reorg: {:?}", + current_hash, + e + ); + *self.latest_tip.lock().unwrap() = None; + return Ok(None); + }, + Ok(Err(e)) => { + log_error!( + self.logger, + "Failed to fetch block for fee estimation: {:?}", + e + ); + return Err(Error::FeerateEstimationUpdateFailed); + }, + Err(e) if idx == 0 => { + log_debug!( + self.logger, + "Timed out fetching cached CBF tip {} during fee estimation, \ + likely due to initial sync or a reorg: {}", + current_hash, + e + ); + *self.latest_tip.lock().unwrap() = None; + return Ok(None); + }, + Err(e) => { + log_error!(self.logger, "Updating fee rate estimates timed out: {}", e); + return Err(Error::FeerateEstimationUpdateTimeout); + }, + }; + + let height = indexed_block.height; + let block = &indexed_block.block; + let weight_kwu = block.weight().to_kwu_floor(); + + // Compute fee rate: (coinbase_output - subsidy) / weight. + // For blocks with zero weight (e.g. coinbase-only in regtest), use the floor rate. + let fee_rate_sat_per_kwu = if weight_kwu == 0 { + MIN_FEERATE_SAT_PER_KWU + } else { + let subsidy = block_subsidy(height); + let revenue = block + .txdata + .first() + .map(|tx| tx.output.iter().map(|o| o.value).sum()) + .unwrap_or(Amount::ZERO); + let block_fees = revenue.checked_sub(subsidy).unwrap_or(Amount::ZERO); + + if block_fees == Amount::ZERO && self.config.network == Network::Bitcoin { + log_error!( + self.logger, + "Failed to retrieve fee rate estimates: zero block fees are disallowed on Mainnet.", + ); + return Err(Error::FeerateEstimationUpdateFailed); + } + + (block_fees.to_sat() / weight_kwu).max(MIN_FEERATE_SAT_PER_KWU) + }; + + block_fee_rates.push(fee_rate_sat_per_kwu); + // Walk backwards through the chain via prev_blockhash. + if height == 0 { + break; + } + current_hash = block.header.prev_blockhash; + } + + if block_fee_rates.is_empty() { + log_error!(self.logger, "No blocks available for fee rate estimation."); + return Err(Error::FeerateEstimationUpdateFailed); + } + + block_fee_rates.sort(); + + let confirmation_targets = get_all_conf_targets(); + let mut new_fee_rate_cache = HashMap::with_capacity(confirmation_targets.len()); + + for target in confirmation_targets { + let num_blocks = get_num_block_defaults_for_target(target); + let base_fee_rate = select_fee_rate_for_target(&block_fee_rates, num_blocks); + let adjusted_fee_rate = apply_post_estimation_adjustments(target, base_fee_rate); + new_fee_rate_cache.insert(target, adjusted_fee_rate); + + log_trace!( + self.logger, + "Fee rate estimation updated for {:?}: {} sats/kwu", + target, + adjusted_fee_rate.to_sat_per_kwu(), + ); + } + + log_debug!( + self.logger, + "CBF fee rate estimation finished in {}ms ({} blocks sampled).", + now.elapsed().as_millis(), + block_fee_rates.len(), + ); + + Ok(Some(new_fee_rate_cache)) + } + + /// Fetch per-target fee rates from an Esplora server. + async fn fee_rate_cache_from_esplora( + &self, client: &esplora_client::AsyncClient, + ) -> Result, Error> { + let timeout = Duration::from_secs( + self.sync_config.timeouts_config.fee_rate_cache_update_timeout_secs, + ); + let estimates = tokio::time::timeout(timeout, client.get_fee_estimates()) + .await + .map_err(|e| { + log_error!(self.logger, "Updating fee rate estimates timed out: {}", e); + Error::FeerateEstimationUpdateTimeout + })? + .map_err(|e| { + log_error!(self.logger, "Failed to retrieve fee rate estimates: {}", e); + Error::FeerateEstimationUpdateFailed + })?; + + if estimates.is_empty() && self.config.network == Network::Bitcoin { + log_error!( + self.logger, + "Failed to retrieve fee rate estimates: empty estimates are disallowed on Mainnet.", + ); + return Err(Error::FeerateEstimationUpdateFailed); + } + + let confirmation_targets = get_all_conf_targets(); + let mut new_fee_rate_cache = HashMap::with_capacity(confirmation_targets.len()); + for target in confirmation_targets { + let num_blocks = get_num_block_defaults_for_target(target); + let converted_estimate_sat_vb = + esplora_client::convert_fee_rate(num_blocks, estimates.clone()) + .map_or(1.0, |converted| converted.max(1.0)); + let fee_rate = FeeRate::from_sat_per_kwu((converted_estimate_sat_vb * 250.0) as u64); + let adjusted_fee_rate = apply_post_estimation_adjustments(target, fee_rate); + new_fee_rate_cache.insert(target, adjusted_fee_rate); + + log_trace!( + self.logger, + "Fee rate estimation updated for {:?}: {} sats/kwu", + target, + adjusted_fee_rate.to_sat_per_kwu(), + ); + } + Ok(new_fee_rate_cache) + } + + /// Fetch per-target fee rates from an Electrum server. + /// + /// Opens a fresh connection for each call because `ElectrumClient` is not `Sync`. + async fn fee_rate_cache_from_electrum( + &self, server_url: &str, + ) -> Result, Error> { + let server_url = server_url.to_owned(); + let confirmation_targets = get_all_conf_targets(); + let per_request_timeout = self.sync_config.timeouts_config.per_request_timeout_secs; + + let raw_estimates: Vec = tokio::time::timeout( + Duration::from_secs( + self.sync_config.timeouts_config.fee_rate_cache_update_timeout_secs, + ), + tokio::task::spawn_blocking(move || { + let electrum_config = electrum_client::ConfigBuilder::new() + .retry(3) + .timeout(Some(per_request_timeout)) + .build(); + let client = electrum_client::Client::from_config(&server_url, electrum_config) + .map_err(|_| Error::FeerateEstimationUpdateFailed)?; + let mut batch = electrum_client::Batch::default(); + for target in confirmation_targets { + batch.estimate_fee(get_num_block_defaults_for_target(target)); + } + client.batch_call(&batch).map_err(|_| Error::FeerateEstimationUpdateFailed) + }), + ) + .await + .map_err(|e| { + log_error!(self.logger, "Updating fee rate estimates timed out: {}", e); + Error::FeerateEstimationUpdateTimeout + })? + .map_err(|_| Error::FeerateEstimationUpdateFailed)? // JoinError + ?; // inner Result + + let confirmation_targets = get_all_conf_targets(); + + if raw_estimates.len() != confirmation_targets.len() + && self.config.network == Network::Bitcoin + { + log_error!( + self.logger, + "Failed to retrieve fee rate estimates: Electrum server didn't return all expected results.", + ); + return Err(Error::FeerateEstimationUpdateFailed); + } + + let mut new_fee_rate_cache = HashMap::with_capacity(confirmation_targets.len()); + for (target, raw_rate) in confirmation_targets.into_iter().zip(raw_estimates.into_iter()) { + // Electrum returns BTC/KvB; fall back to 1 sat/vb (= 0.00001 BTC/KvB) on failure. + let fee_rate_btc_per_kvb = + raw_rate.as_f64().map_or(0.00001_f64, |v: f64| v.max(0.00001)); + // Convert BTC/KvB → sat/kwu: multiply by 25_000_000 (= 10^8 / 4). + let fee_rate = + FeeRate::from_sat_per_kwu((fee_rate_btc_per_kvb * 25_000_000.0).round() as u64); + let adjusted_fee_rate = apply_post_estimation_adjustments(target, fee_rate); + new_fee_rate_cache.insert(target, adjusted_fee_rate); + + log_trace!( + self.logger, + "Fee rate estimation updated for {:?}: {} sats/kwu", + target, + adjusted_fee_rate.to_sat_per_kwu(), + ); + } + Ok(new_fee_rate_cache) + } + + /// Broadcast a package of transactions via the P2P network. + pub(crate) async fn process_broadcast_package(&self, package: Vec) { + let Ok(requester) = self.requester() else { return }; + + for tx in package { + let txid = tx.compute_txid(); + let tx_bytes = tx.encode(); + let timeout_fut = tokio::time::timeout( + Duration::from_secs(self.sync_config.timeouts_config.tx_broadcast_timeout_secs), + requester.broadcast_tx(tx), + ); + match timeout_fut.await { + Ok(res) => match res { + Ok(wtxid) => { + log_trace!( + self.logger, + "Successfully broadcast transaction {} (wtxid: {})", + txid, + wtxid + ); + }, + Err(e) => { + log_error!( + self.logger, + "Failed to broadcast transaction {}: {:?}", + txid, + e + ); + log_trace!( + self.logger, + "Failed broadcast transaction bytes: {}", + log_bytes!(tx_bytes) + ); + }, + }, + Err(e) => { + log_error!( + self.logger, + "Failed to broadcast transaction due to timeout {}: {}", + txid, + e + ); + log_trace!( + self.logger, + "Failed broadcast transaction bytes: {}", + log_bytes!(tx_bytes) + ); + }, + } + } + } +} + +/// Record the current timestamp in a `NodeMetrics` field and persist the metrics. +fn update_node_metrics_timestamp( + node_metrics: &RwLock, kv_store: &DynStore, logger: &Logger, + setter: impl FnOnce(&mut NodeMetrics, Option), +) -> Result<(), Error> { + let unix_time_secs_opt = SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); + let mut locked = node_metrics.write().unwrap(); + setter(&mut locked, unix_time_secs_opt); + write_node_metrics(&*locked, kv_store, logger)?; + Ok(()) +} + +/// Fetch a block by hash and call `transactions_confirmed` on each confirmable. +async fn confirm_block_transactions( + requester: &Requester, block_hash: BlockHash, height: u32, + confirmables: &[&(dyn Confirm + Sync + Send)], logger: &Logger, +) -> Result<(), Error> { + let indexed_block = requester.get_block(block_hash).await.map_err(|e| { + log_error!(logger, "Failed to fetch block {}: {:?}", block_hash, e); + Error::TxSyncFailed + })?; + let block = &indexed_block.block; + let header = &block.header; + let txdata: Vec<(usize, &Transaction)> = block.txdata.iter().enumerate().collect(); + if !txdata.is_empty() { + for confirmable in confirmables { + confirmable.transactions_confirmed(header, &txdata, height); + } + } + Ok(()) +} + +/// Compute the block subsidy (mining reward before fees) at the given block height. +fn block_subsidy(height: u32) -> Amount { + let halvings = height / SUBSIDY_HALVING_INTERVAL; + if halvings >= 64 { + return Amount::ZERO; + } + let base = Amount::ONE_BTC.to_sat() * 50; + Amount::from_sat(base >> halvings) +} + +/// Select a fee rate from sorted block fee rates based on confirmation urgency. +/// +/// For urgent targets (1 block), uses the highest observed fee rate. +/// For medium targets (2-6 blocks), uses the 75th percentile. +/// For standard targets (7-12 blocks), uses the median. +/// For low-urgency targets (13+ blocks), uses the 25th percentile. +fn select_fee_rate_for_target(sorted_rates: &[u64], num_blocks: usize) -> FeeRate { + if sorted_rates.is_empty() { + return FeeRate::from_sat_per_kwu(MIN_FEERATE_SAT_PER_KWU); + } + + let len = sorted_rates.len(); + let idx = if num_blocks <= 1 { + len - 1 + } else if num_blocks <= 6 { + (len * 3) / 4 + } else if num_blocks <= 12 { + len / 2 + } else { + len / 4 + }; + + FeeRate::from_sat_per_kwu(sorted_rates[idx.min(len - 1)]) +} + +#[cfg(test)] +mod tests { + use bitcoin::constants::SUBSIDY_HALVING_INTERVAL; + use bitcoin::{Amount, FeeRate}; + + use super::{block_subsidy, select_fee_rate_for_target, MIN_FEERATE_SAT_PER_KWU}; + use crate::fee_estimator::{ + apply_post_estimation_adjustments, get_all_conf_targets, get_num_block_defaults_for_target, + }; + + #[test] + fn select_fee_rate_empty_returns_floor() { + let rate = select_fee_rate_for_target(&[], 1); + assert_eq!(rate, FeeRate::from_sat_per_kwu(MIN_FEERATE_SAT_PER_KWU)); + } + + #[test] + fn select_fee_rate_single_element_returns_it_for_all_buckets() { + let rates = [5000u64]; + // Every urgency bucket should return the single available rate. + for num_blocks in [1, 3, 6, 12, 144, 1008] { + let rate = select_fee_rate_for_target(&rates, num_blocks); + assert_eq!( + rate, + FeeRate::from_sat_per_kwu(5000), + "num_blocks={} should return the only available rate", + num_blocks, + ); + } + } + + #[test] + fn select_fee_rate_picks_correct_percentile() { + // 6 sorted rates: indices 0..5 + let rates = [100, 200, 300, 400, 500, 600]; + // 1-block (most urgent): highest → index 5 → 600 + assert_eq!(select_fee_rate_for_target(&rates, 1), FeeRate::from_sat_per_kwu(600)); + // 6-block (medium): 75th percentile → (6*3)/4 = 4 → 500 + assert_eq!(select_fee_rate_for_target(&rates, 6), FeeRate::from_sat_per_kwu(500)); + // 12-block (standard): median → 6/2 = 3 → 400 + assert_eq!(select_fee_rate_for_target(&rates, 12), FeeRate::from_sat_per_kwu(400)); + // 144-block (low): 25th percentile → 6/4 = 1 → 200 + assert_eq!(select_fee_rate_for_target(&rates, 144), FeeRate::from_sat_per_kwu(200)); + } + + #[test] + fn select_fee_rate_monotonic_urgency() { + // More urgent targets should never produce lower rates than less urgent ones. + let rates = [250, 500, 1000, 2000, 4000, 8000]; + let urgent = select_fee_rate_for_target(&rates, 1); + let medium = select_fee_rate_for_target(&rates, 6); + let standard = select_fee_rate_for_target(&rates, 12); + let low = select_fee_rate_for_target(&rates, 144); + + assert!( + urgent >= medium, + "urgent ({}) >= medium ({})", + urgent.to_sat_per_kwu(), + medium.to_sat_per_kwu() + ); + assert!( + medium >= standard, + "medium ({}) >= standard ({})", + medium.to_sat_per_kwu(), + standard.to_sat_per_kwu() + ); + assert!( + standard >= low, + "standard ({}) >= low ({})", + standard.to_sat_per_kwu(), + low.to_sat_per_kwu() + ); + } + + #[test] + fn uniform_rates_match_naive_single_rate() { + // When all blocks have the same fee rate (like the old single-block + // approach), every target should select that same base rate. This + // proves the optimized multi-block approach is backwards-compatible. + + let uniform_rate = 3000u64; + let rates = [uniform_rate; 6]; + for target in get_all_conf_targets() { + let num_blocks = get_num_block_defaults_for_target(target); + let optimized = select_fee_rate_for_target(&rates, num_blocks); + let naive = FeeRate::from_sat_per_kwu(uniform_rate); + assert_eq!( + optimized, naive, + "For target {:?} (num_blocks={}), optimized rate should match naive single-rate", + target, num_blocks, + ); + + // Also verify the post-estimation adjustments produce the same + // result for both approaches. + let adjusted_optimized = apply_post_estimation_adjustments(target, optimized); + let adjusted_naive = apply_post_estimation_adjustments(target, naive); + assert_eq!(adjusted_optimized, adjusted_naive); + } + } + + #[test] + fn block_subsidy_genesis() { + assert_eq!(block_subsidy(0), Amount::from_sat(50 * 100_000_000)); + } + + #[test] + fn block_subsidy_first_halving() { + assert_eq!(block_subsidy(SUBSIDY_HALVING_INTERVAL), Amount::from_sat(25 * 100_000_000)); + } + + #[test] + fn block_subsidy_second_halving() { + assert_eq!(block_subsidy(SUBSIDY_HALVING_INTERVAL * 2), Amount::from_sat(1_250_000_000)); + } + + #[test] + fn block_subsidy_exhausted_after_64_halvings() { + assert_eq!(block_subsidy(SUBSIDY_HALVING_INTERVAL * 64), Amount::ZERO); + assert_eq!(block_subsidy(SUBSIDY_HALVING_INTERVAL * 100), Amount::ZERO); + } + + #[test] + fn select_fee_rate_two_elements() { + let rates = [1000, 5000]; + // 1-block: index 1 (highest) → 5000 + assert_eq!(select_fee_rate_for_target(&rates, 1), FeeRate::from_sat_per_kwu(5000)); + // 6-block: (2*3)/4 = 1 → 5000 + assert_eq!(select_fee_rate_for_target(&rates, 6), FeeRate::from_sat_per_kwu(5000)); + // 12-block: 2/2 = 1 → 5000 + assert_eq!(select_fee_rate_for_target(&rates, 12), FeeRate::from_sat_per_kwu(5000)); + // 144-block: 2/4 = 0 → 1000 + assert_eq!(select_fee_rate_for_target(&rates, 144), FeeRate::from_sat_per_kwu(1000)); + } + + #[test] + fn select_fee_rate_all_targets_use_valid_indices() { + for size in 1..=6 { + let rates: Vec = (1..=size).map(|i| i as u64 * 1000).collect(); + for target in get_all_conf_targets() { + let num_blocks = get_num_block_defaults_for_target(target); + let _ = select_fee_rate_for_target(&rates, num_blocks); + } + } + } +} diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 49c011a78d..b896ba6fbf 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -6,6 +6,7 @@ // accordance with one or both of these licenses. pub(crate) mod bitcoind; +mod cbf; mod electrum; mod esplora; @@ -17,11 +18,12 @@ use bitcoin::{Script, Txid}; use lightning::chain::{BestBlock, Filter}; use crate::chain::bitcoind::{BitcoindChainSource, UtxoSourceClient}; +use crate::chain::cbf::CbfChainSource; use crate::chain::electrum::ElectrumChainSource; use crate::chain::esplora::EsploraChainSource; use crate::config::{ - BackgroundSyncConfig, BitcoindRestClientConfig, Config, ElectrumSyncConfig, EsploraSyncConfig, - WALLET_SYNC_INTERVAL_MINIMUM_SECS, + BackgroundSyncConfig, BitcoindRestClientConfig, CbfSyncConfig, Config, ElectrumSyncConfig, + EsploraSyncConfig, WALLET_SYNC_INTERVAL_MINIMUM_SECS, }; use crate::fee_estimator::OnchainFeeEstimator; use crate::logger::{log_debug, log_info, log_trace, LdkLogger, Logger}; @@ -82,6 +84,20 @@ impl WalletSyncStatus { } } +/// Optional external fee estimation backend for the CBF chain source. +/// +/// By default CBF derives fee rates from recent blocks' coinbase outputs. +/// Setting an external source provides more accurate, per-target estimates +/// from a mempool-aware server. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +pub enum FeeSourceConfig { + /// Use an Esplora HTTP server for fee rate estimation. + Esplora(String), + /// Use an Electrum server for fee rate estimation. + Electrum(String), +} + pub(crate) struct ChainSource { kind: ChainSourceKind, registered_txids: Mutex>, @@ -93,6 +109,7 @@ enum ChainSourceKind { Esplora(EsploraChainSource), Electrum(ElectrumChainSource), Bitcoind(BitcoindChainSource), + Cbf(CbfChainSource), } impl ChainSource { @@ -184,11 +201,35 @@ impl ChainSource { (Self { kind, registered_txids, tx_broadcaster, logger }, best_block) } + pub(crate) fn new_cbf( + peers: Vec, sync_config: CbfSyncConfig, fee_source_config: Option, + fee_estimator: Arc, tx_broadcaster: Arc, + kv_store: Arc, config: Arc, logger: Arc, + node_metrics: Arc>, + ) -> Result<(Self, Option), Error> { + let cbf_chain_source = CbfChainSource::new( + peers, + sync_config, + fee_source_config, + fee_estimator, + kv_store, + config, + Arc::clone(&logger), + node_metrics, + )?; + let kind = ChainSourceKind::Cbf(cbf_chain_source); + let registered_txids = Mutex::new(Vec::new()); + Ok((Self { kind, registered_txids, tx_broadcaster, logger }, None)) + } + pub(crate) fn start(&self, runtime: Arc) -> Result<(), Error> { match &self.kind { ChainSourceKind::Electrum(electrum_chain_source) => { electrum_chain_source.start(runtime)? }, + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source.start(runtime); + }, _ => { // Nothing to do for other chain sources. }, @@ -199,6 +240,9 @@ impl ChainSource { pub(crate) fn stop(&self) { match &self.kind { ChainSourceKind::Electrum(electrum_chain_source) => electrum_chain_source.stop(), + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source.stop(); + }, _ => { // Nothing to do for other chain sources. }, @@ -210,6 +254,7 @@ impl ChainSource { ChainSourceKind::Bitcoind(bitcoind_chain_source) => { Some(bitcoind_chain_source.as_utxo_source()) }, + ChainSourceKind::Cbf { .. } => None, _ => None, } } @@ -223,6 +268,7 @@ impl ChainSource { ChainSourceKind::Esplora(_) => true, ChainSourceKind::Electrum { .. } => true, ChainSourceKind::Bitcoind { .. } => false, + ChainSourceKind::Cbf { .. } => true, } } @@ -289,6 +335,28 @@ impl ChainSource { ) .await }, + ChainSourceKind::Cbf(cbf_chain_source) => { + if let Some(background_sync_config) = + cbf_chain_source.sync_config.background_sync_config.as_ref() + { + self.start_tx_based_sync_loop( + stop_sync_receiver, + onchain_wallet, + channel_manager, + chain_monitor, + output_sweeper, + background_sync_config, + Arc::clone(&self.logger), + ) + .await + } else { + log_info!( + self.logger, + "Background syncing is disabled. Manual syncing required for onchain wallet, lightning wallet, and fee rate updates.", + ); + return; + } + }, } } @@ -368,6 +436,9 @@ impl ChainSource { // `ChainPoller`. So nothing to do here. unreachable!("Onchain wallet will be synced via chain polling") }, + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source.sync_onchain_wallet(onchain_wallet).await + }, } } @@ -393,6 +464,11 @@ impl ChainSource { // `ChainPoller`. So nothing to do here. unreachable!("Lightning wallet will be synced via chain polling") }, + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source + .sync_lightning_wallet(channel_manager, chain_monitor, output_sweeper) + .await + }, } } @@ -421,6 +497,10 @@ impl ChainSource { ) .await }, + ChainSourceKind::Cbf { .. } => { + // In CBF mode we sync wallets via compact block filters. + unreachable!("Listeners will be synced via compact block filter syncing") + }, } } @@ -435,6 +515,9 @@ impl ChainSource { ChainSourceKind::Bitcoind(bitcoind_chain_source) => { bitcoind_chain_source.update_fee_rate_estimates().await }, + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source.update_fee_rate_estimates().await + }, } } @@ -463,6 +546,9 @@ impl ChainSource { ChainSourceKind::Bitcoind(bitcoind_chain_source) => { bitcoind_chain_source.process_broadcast_package(next_package).await }, + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source.process_broadcast_package(next_package).await + }, } } } @@ -481,6 +567,9 @@ impl Filter for ChainSource { electrum_chain_source.register_tx(txid, script_pubkey) }, ChainSourceKind::Bitcoind { .. } => (), + ChainSourceKind::Cbf(cbf_chain_source) => { + cbf_chain_source.register_tx(txid, script_pubkey) + }, } } fn register_output(&self, output: lightning::chain::WatchedOutput) { @@ -492,6 +581,7 @@ impl Filter for ChainSource { electrum_chain_source.register_output(output) }, ChainSourceKind::Bitcoind { .. } => (), + ChainSourceKind::Cbf(cbf_chain_source) => cbf_chain_source.register_output(output), } } } diff --git a/src/config.rs b/src/config.rs index 71e4d23141..893c734d98 100644 --- a/src/config.rs +++ b/src/config.rs @@ -482,6 +482,53 @@ impl Default for ElectrumSyncConfig { } } +/// Configuration for syncing via BIP 157 compact block filters. +/// +/// Background syncing is enabled by default, using the default values specified in +/// [`BackgroundSyncConfig`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +pub struct CbfSyncConfig { + /// Background sync configuration. + /// + /// If set to `None`, background syncing will be disabled. Users will need to manually + /// sync via [`Node::sync_wallets`] for the wallets and fee rate updates. + /// + /// [`Node::sync_wallets`]: crate::Node::sync_wallets + pub background_sync_config: Option, + /// Sync timeouts configuration. + pub timeouts_config: SyncTimeoutsConfig, + /// Peer response timeout in seconds for the bip157 P2P node. + /// + /// If a peer does not respond within this duration, the connection may be dropped. + /// Higher values are recommended for slow peers or when downloading many blocks. + /// + /// Defaults to 30 seconds. + pub response_timeout_secs: u64, + /// Number of peers that must agree on filter headers before they are accepted. + /// + /// Higher values increase security against malicious peers serving invalid compact block + /// filters, at the cost of slower sync times. Must be between 1 and 15. + /// + /// As recommended by BIP 157, clients should connect to multiple peers to mitigate the risk + /// of downloading incorrect filter headers. Setting this to 1 means filter headers from a + /// single peer are trusted without cross-validation. + /// + /// Defaults to 2. + pub required_peers: u8, +} + +impl Default for CbfSyncConfig { + fn default() -> Self { + Self { + background_sync_config: Some(BackgroundSyncConfig::default()), + timeouts_config: SyncTimeoutsConfig::default(), + response_timeout_secs: 30, + required_peers: 2, + } + } +} + /// Configuration for syncing with Bitcoin Core backend via REST. #[derive(Debug, Clone)] pub struct BitcoindRestClientConfig { diff --git a/src/connection.rs b/src/connection.rs index a1d24e36da..9110ed0d96 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -6,6 +6,7 @@ // accordance with one or both of these licenses. use std::collections::hash_map::{self, HashMap}; +use std::net::ToSocketAddrs; use std::ops::Deref; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -14,7 +15,7 @@ use bitcoin::secp256k1::PublicKey; use lightning::ln::msgs::SocketAddress; use crate::config::TorConfig; -use crate::logger::{log_debug, log_error, log_info, LdkLogger}; +use crate::logger::{log_error, log_info, LdkLogger}; use crate::types::{KeysManager, PeerManager}; use crate::Error; @@ -55,14 +56,6 @@ where pub(crate) async fn do_connect_peer( &self, node_id: PublicKey, addr: SocketAddress, - ) -> Result<(), Error> { - let res = self.do_connect_peer_internal(node_id, addr).await; - self.propagate_result_to_subscribers(&node_id, res); - res - } - - async fn do_connect_peer_internal( - &self, node_id: PublicKey, addr: SocketAddress, ) -> Result<(), Error> { // First, we check if there is already an outbound connection in flight, if so, we just // await on the corresponding watch channel. The task driving the connection future will @@ -78,14 +71,15 @@ where log_info!(self.logger, "Connecting to peer: {}@{}", node_id, addr); - match addr { + let res = match addr { SocketAddress::OnionV2(old_onion_addr) => { log_error!( - self.logger, - "Failed to resolve network address {:?}: Resolution of OnionV2 addresses is currently unsupported.", - old_onion_addr - ); - Err(Error::InvalidSocketAddress) + self.logger, + "Failed to resolve network address {:?}: Resolution of OnionV2 addresses is currently unsupported.", + old_onion_addr + ); + self.propagate_result_to_subscribers(&node_id, Err(Error::InvalidSocketAddress)); + return Err(Error::InvalidSocketAddress); }, SocketAddress::OnionV3 { .. } => { let proxy_config = self.tor_proxy_config.as_ref().ok_or_else(|| { @@ -94,66 +88,53 @@ where "Failed to resolve network address {:?}: Tor usage is not configured.", addr ); + self.propagate_result_to_subscribers( + &node_id, + Err(Error::InvalidSocketAddress), + ); Error::InvalidSocketAddress })?; - let resolved_addrs: Vec<_> = - tokio::net::lookup_host(proxy_config.proxy_address.to_string()) - .await - .map_err(|e| { - log_error!( - self.logger, - "Failed to resolve Tor proxy network address {}: {}", - proxy_config.proxy_address, - e - ); - Error::InvalidSocketAddress - })? - .collect(); - - if resolved_addrs.is_empty() { - log_error!( - self.logger, - "Failed to resolve Tor proxy network address {}", - proxy_config.proxy_address - ); - return Err(Error::InvalidSocketAddress); - } - - let mut res = Err(Error::ConnectionFailed); - let mut had_failures = false; - for proxy_addr in resolved_addrs { - let connection_future = lightning_net_tokio::tor_connect_outbound( - Arc::clone(&self.peer_manager), - node_id, - addr.clone(), - proxy_addr, - Arc::clone(&self.keys_manager), - ); - res = self.await_connection(connection_future, node_id, addr.clone()).await; - if res.is_ok() { - if had_failures { - log_info!( - self.logger, - "Successfully connected to peer {}@{} via resolved proxy address {} after previous attempts failed.", - node_id, addr, proxy_addr - ); - } - break; - } - had_failures = true; - log_debug!( - self.logger, - "Failed to connect to peer {}@{} via resolved proxy address {}.", - node_id, - addr, - proxy_addr - ); - } - res + let proxy_addr = proxy_config + .proxy_address + .to_socket_addrs() + .map_err(|e| { + log_error!( + self.logger, + "Failed to resolve Tor proxy network address {}: {}", + proxy_config.proxy_address, + e + ); + self.propagate_result_to_subscribers( + &node_id, + Err(Error::InvalidSocketAddress), + ); + Error::InvalidSocketAddress + })? + .next() + .ok_or_else(|| { + log_error!( + self.logger, + "Failed to resolve Tor proxy network address {}", + proxy_config.proxy_address + ); + self.propagate_result_to_subscribers( + &node_id, + Err(Error::InvalidSocketAddress), + ); + Error::InvalidSocketAddress + })?; + let connection_future = lightning_net_tokio::tor_connect_outbound( + Arc::clone(&self.peer_manager), + node_id, + addr.clone(), + proxy_addr, + Arc::clone(&self.keys_manager), + ); + self.await_connection(connection_future, node_id, addr).await }, _ => { - let resolved_addrs: Vec<_> = tokio::net::lookup_host(addr.to_string()) - .await + let socket_addr = addr + .to_socket_addrs() .map_err(|e| { log_error!( self.logger, @@ -161,46 +142,33 @@ where addr, e ); + self.propagate_result_to_subscribers( + &node_id, + Err(Error::InvalidSocketAddress), + ); Error::InvalidSocketAddress })? - .collect(); + .next() + .ok_or_else(|| { + log_error!(self.logger, "Failed to resolve network address {}", addr); + self.propagate_result_to_subscribers( + &node_id, + Err(Error::InvalidSocketAddress), + ); + Error::InvalidSocketAddress + })?; + let connection_future = lightning_net_tokio::connect_outbound( + Arc::clone(&self.peer_manager), + node_id, + socket_addr, + ); + self.await_connection(connection_future, node_id, addr).await + }, + }; - if resolved_addrs.is_empty() { - log_error!(self.logger, "Failed to resolve network address {}", addr); - return Err(Error::InvalidSocketAddress); - } + self.propagate_result_to_subscribers(&node_id, res); - let mut res = Err(Error::ConnectionFailed); - let mut had_failures = false; - for socket_addr in resolved_addrs { - let connection_future = lightning_net_tokio::connect_outbound( - Arc::clone(&self.peer_manager), - node_id, - socket_addr, - ); - res = self.await_connection(connection_future, node_id, addr.clone()).await; - if res.is_ok() { - if had_failures { - log_info!( - self.logger, - "Successfully connected to peer {}@{} via resolved address {} after previous attempts failed.", - node_id, addr, socket_addr - ); - } - break; - } - had_failures = true; - log_debug!( - self.logger, - "Failed to connect to peer {}@{} via resolved address {}.", - node_id, - addr, - socket_addr - ); - } - res - }, - } + res } async fn await_connection( diff --git a/src/ffi/types.rs b/src/ffi/types.rs index 5a1420882e..57d51f7187 100644 --- a/src/ffi/types.rs +++ b/src/ffi/types.rs @@ -142,7 +142,9 @@ impl VssClientHeaderProvider for VssHeaderProviderAdapter { } use crate::builder::sanitize_alias; -pub use crate::config::{default_config, ElectrumSyncConfig, EsploraSyncConfig, TorConfig}; +pub use crate::config::{ + default_config, CbfSyncConfig, ElectrumSyncConfig, EsploraSyncConfig, TorConfig, +}; pub use crate::entropy::{generate_entropy_mnemonic, NodeEntropy, WordCount}; use crate::error::Error; pub use crate::liquidity::LSPS1OrderStatus; diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 94e8360fc4..c69e5685d1 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -684,6 +684,7 @@ impl SqliteStoreInner { #[cfg(test)] mod tests { use super::*; + use crate::io::test_utils::{ do_read_write_remove_list_persist, do_test_store, random_storage_path, }; diff --git a/src/lib.rs b/src/lib.rs index 2ac4697e81..1f6df49fc2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,6 +108,7 @@ mod types; mod wallet; use std::default::Default; +use std::net::ToSocketAddrs; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; #[cfg(cycle_tests)] @@ -128,6 +129,7 @@ pub use builder::BuildError; #[cfg(not(feature = "uniffi"))] pub use builder::NodeBuilder as Builder; use chain::ChainSource; +pub use chain::FeeSourceConfig; use config::{ default_user_config, may_announce_channel, AsyncPaymentsRole, ChannelConfig, Config, LNURL_AUTH_TIMEOUT_SECS, NODE_ANN_BCAST_INTERVAL, PEER_RECONNECTION_INTERVAL, @@ -360,29 +362,28 @@ impl Node { let peer_manager_connection_handler = Arc::clone(&self.peer_manager); let listening_logger = Arc::clone(&self.logger); - let logger = Arc::clone(&listening_logger); - let listening_addrs = listening_addresses.clone(); - let listeners = self.runtime.block_on(async move { - let mut bind_addrs = Vec::with_capacity(listening_addrs.len()); + let mut bind_addrs = Vec::with_capacity(listening_addresses.len()); - for listening_addr in &listening_addrs { - let resolved = - tokio::net::lookup_host(listening_addr.to_string()).await.map_err(|e| { - log_error!( - logger, - "Unable to resolve listening address: {:?}. Error details: {}", - listening_addr, - e, - ); - Error::InvalidSocketAddress - })?; - bind_addrs.extend(resolved); - } + for listening_addr in listening_addresses { + let resolved_address = listening_addr.to_socket_addrs().map_err(|e| { + log_error!( + self.logger, + "Unable to resolve listening address: {:?}. Error details: {}", + listening_addr, + e, + ); + Error::InvalidSocketAddress + })?; + bind_addrs.extend(resolved_address); + } + + let logger = Arc::clone(&listening_logger); + let listeners = self.runtime.block_on(async move { let mut listeners = Vec::new(); // Try to bind to all addresses - for addr in &bind_addrs { + for addr in &*bind_addrs { match tokio::net::TcpListener::bind(addr).await { Ok(listener) => { log_trace!(logger, "Listener bound to {}", addr); diff --git a/src/lnurl_auth.rs b/src/lnurl_auth.rs index 1f95b77b17..1a0def47c5 100644 --- a/src/lnurl_auth.rs +++ b/src/lnurl_auth.rs @@ -96,13 +96,6 @@ impl LnurlAuth { let domain = url.base_url(); - // Enforce HTTPS for non-localhost URLs per LNURL spec. - let is_localhost = domain == "localhost" || domain == "127.0.0.1" || domain == "[::1]"; - if url.scheme() != "https" && !is_localhost { - log_error!(self.logger, "LNURL-auth URL must use HTTPS for non-localhost domains"); - return Err(Error::InvalidLnurl); - } - // get query parameters for k1 and tag let query_params: std::collections::HashMap<_, _> = url.query_pairs().collect(); @@ -142,7 +135,7 @@ impl LnurlAuth { let auth_url = format!("{lnurl_auth_url}&sig={signature}&key={linking_public_key}"); log_debug!(self.logger, "Submitting LNURL-auth response"); - let request = bitreq::get(&auth_url).with_max_redirects(0); + let request = bitreq::get(&auth_url); let auth_response = self.client.send_async(request).await.map_err(|e| { log_error!(self.logger, "Failed to submit LNURL-auth response: {e}"); Error::LnurlAuthFailed diff --git a/src/payment/asynchronous/rate_limiter.rs b/src/payment/asynchronous/rate_limiter.rs index bf12508927..671b1dc72a 100644 --- a/src/payment/asynchronous/rate_limiter.rs +++ b/src/payment/asynchronous/rate_limiter.rs @@ -23,8 +23,6 @@ pub(crate) struct RateLimiter { max_idle: Duration, } -const MAX_USERS: usize = 10_000; - struct Bucket { tokens: u32, last_refill: Instant, @@ -38,19 +36,10 @@ impl RateLimiter { pub(crate) fn allow(&mut self, user_id: &[u8]) -> bool { let now = Instant::now(); - let is_new_user = !self.users.contains_key(user_id); - - if is_new_user { - self.garbage_collect(self.max_idle); - if self.users.len() >= MAX_USERS { - return false; - } - } + let entry = self.users.entry(user_id.to_vec()); + let is_new_user = matches!(entry, std::collections::hash_map::Entry::Vacant(_)); - let bucket = self - .users - .entry(user_id.to_vec()) - .or_insert(Bucket { tokens: self.capacity, last_refill: now }); + let bucket = entry.or_insert(Bucket { tokens: self.capacity, last_refill: now }); let elapsed = now.duration_since(bucket.last_refill); let tokens_to_add = (elapsed.as_secs_f64() / self.refill_interval.as_secs_f64()) as u32; @@ -67,6 +56,11 @@ impl RateLimiter { false }; + // Each time a new user is added, we take the opportunity to clean up old rate limits. + if is_new_user { + self.garbage_collect(self.max_idle); + } + allow } diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 0e80a46dbd..8d4f22cbb8 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -122,6 +122,28 @@ impl Wallet { self.inner.lock().unwrap().start_sync_with_revealed_spks().build() } + pub(crate) fn get_spks_for_cbf_sync(&self, stop_gap: usize) -> Vec { + let wallet = self.inner.lock().unwrap(); + let mut scripts: Vec = + wallet.spk_index().revealed_spks(..).map(|((_, _), spk)| spk).collect(); + + // For first sync when no scripts have been revealed yet, generate + // lookahead scripts up to the stop gap for both keychains. + if scripts.is_empty() { + for keychain in [KeychainKind::External, KeychainKind::Internal] { + for idx in 0..stop_gap as u32 { + scripts.push(wallet.peek_address(keychain, idx).address.script_pubkey()); + } + } + } + + scripts + } + + pub(crate) fn latest_checkpoint(&self) -> bdk_chain::CheckPoint { + self.inner.lock().unwrap().latest_checkpoint() + } + pub(crate) fn get_cached_txs(&self) -> Vec> { self.inner.lock().unwrap().tx_graph().full_txs().map(|tx_node| tx_node.tx).collect() } @@ -1155,9 +1177,15 @@ impl Wallet { let kind = PaymentKind::Onchain { txid, status: confirmation_status }; - let fee = locked_wallet.calculate_fee(tx).unwrap_or(Amount::ZERO); + let fee = match locked_wallet.calculate_fee(tx) { + Ok(fee) => Some(fee), + Err(e) => { + log_error!(self.logger, "Failed to calculate fee for tx {}: {:?}", txid, e); + None + }, + }; let (sent, received) = locked_wallet.sent_and_received(tx); - let fee_sat = fee.to_sat(); + let fee_sat = fee.map_or(0, |f| f.to_sat()); let (direction, amount_msat) = if sent > received { ( @@ -1180,7 +1208,7 @@ impl Wallet { payment_id, kind, amount_msat, - Some(fee_sat * 1000), + fee.map(|f| f.to_sat() * 1000), direction, payment_status, ) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4f68f9825f..4fd17499c1 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,6 +7,8 @@ #![cfg(any(test, cln_test, lnd_test, vss_test))] #![allow(dead_code)] +#![allow(unused_imports)] +#![allow(unused_macros)] pub(crate) mod logging; @@ -27,7 +29,10 @@ use bitcoin::{ use electrsd::corepc_node::{Client as BitcoindClient, Node as BitcoinD}; use electrsd::{corepc_node, ElectrsD}; use electrum_client::ElectrumApi; -use ldk_node::config::{AsyncPaymentsRole, Config, ElectrumSyncConfig, EsploraSyncConfig}; +use ldk_node::config::{ + AsyncPaymentsRole, CbfSyncConfig, Config, ElectrumSyncConfig, EsploraSyncConfig, + SyncTimeoutsConfig, +}; use ldk_node::entropy::{generate_entropy_mnemonic, NodeEntropy}; use ldk_node::io::sqlite_store::SqliteStore; use ldk_node::payment::{PaymentDirection, PaymentKind, PaymentStatus}; @@ -48,6 +53,17 @@ use rand::distr::Alphanumeric; use rand::{rng, Rng}; use serde_json::{json, Value}; +macro_rules! skip_if_cbf { + ($chain_source:expr) => { + if matches!($chain_source, TestChainSource::Cbf(_)) { + println!("Skipping test: not compatible with CBF chain source"); + return; + } + }; +} + +pub(crate) use skip_if_cbf; + macro_rules! expect_event { ($node:expr, $event_type:ident) => {{ match $node.next_event_async().await { @@ -223,6 +239,11 @@ pub(crate) fn setup_bitcoind_and_electrsd() -> (BitcoinD, ElectrsD) { let mut bitcoind_conf = corepc_node::Conf::default(); bitcoind_conf.network = "regtest"; bitcoind_conf.args.push("-rest"); + + bitcoind_conf.p2p = corepc_node::P2P::Yes; + bitcoind_conf.args.push("-blockfilterindex=1"); + bitcoind_conf.args.push("-peerblockfilters=1"); + let bitcoind = BitcoinD::with_conf(bitcoind_exe, &bitcoind_conf).unwrap(); let electrs_exe = env::var("ELECTRS_EXE") @@ -239,7 +260,16 @@ pub(crate) fn setup_bitcoind_and_electrsd() -> (BitcoinD, ElectrsD) { pub(crate) fn random_chain_source<'a>( bitcoind: &'a BitcoinD, electrsd: &'a ElectrsD, ) -> TestChainSource<'a> { - let r = rand::random_range(0..3); + // Allow forcing a specific backend via LDK_TEST_CHAIN_SOURCE env var. + // Valid values: "esplora", "electrum", "bitcoind-rpc", "bitcoind-rest", "cbf" + let r = match std::env::var("LDK_TEST_CHAIN_SOURCE").ok().as_deref() { + Some("esplora") => 0, + Some("electrum") => 1, + Some("bitcoind-rpc") => 2, + Some("bitcoind-rest") => 3, + Some("cbf") => 4, + _ => rand::random_range(0..5), + }; match r { 0 => { println!("Randomly setting up Esplora chain syncing..."); @@ -257,6 +287,10 @@ pub(crate) fn random_chain_source<'a>( println!("Randomly setting up Bitcoind REST chain syncing..."); TestChainSource::BitcoindRestSync(bitcoind) }, + 4 => { + println!("Randomly setting up CBF compact block filter syncing..."); + TestChainSource::Cbf(bitcoind) + }, _ => unreachable!(), } } @@ -324,6 +358,7 @@ pub(crate) enum TestChainSource<'a> { Electrum(&'a ElectrsD), BitcoindRpcSync(&'a BitcoinD), BitcoindRestSync(&'a BitcoinD), + Cbf(&'a BitcoinD), } #[derive(Clone, Copy)] @@ -461,6 +496,24 @@ pub(crate) fn setup_node(chain_source: &TestChainSource, config: TestConfig) -> rpc_password, ); }, + TestChainSource::Cbf(bitcoind) => { + let p2p_socket = bitcoind.params.p2p_socket.expect("P2P must be enabled for CBF"); + let peer_addr = format!("{}", p2p_socket); + let timeouts_config = SyncTimeoutsConfig { + onchain_wallet_sync_timeout_secs: 3, + lightning_wallet_sync_timeout_secs: 3, + fee_rate_cache_update_timeout_secs: 3, + tx_broadcast_timeout_secs: 3, + per_request_timeout_secs: 3, + }; + let sync_config = CbfSyncConfig { + background_sync_config: None, + timeouts_config, + required_peers: 1, + ..Default::default() + }; + builder.set_chain_source_cbf(vec![peer_addr], Some(sync_config), None); + }, } match &config.log_writer { @@ -495,7 +548,10 @@ pub(crate) fn setup_node(chain_source: &TestChainSource, config: TestConfig) -> node.start().unwrap(); assert!(node.status().is_running); - assert!(node.status().latest_fee_rate_cache_update_timestamp.is_some()); + + if !matches!(chain_source, TestChainSource::Cbf(_)) { + assert!(node.status().latest_fee_rate_cache_update_timestamp.is_some()); + } node } @@ -569,7 +625,9 @@ pub(crate) async fn wait_for_outpoint_spend(electrs: &E, outpoin let tx = electrs.transaction_get(&outpoint.txid).unwrap(); let txout_script = tx.output.get(outpoint.vout as usize).unwrap().clone().script_pubkey; - let is_spent = !electrs.script_get_history(&txout_script).unwrap().is_empty(); + // An output's script will have at least 1 history entry (the tx that created it). + // When the output is spent, there will be at least 2 entries (creating + spending tx). + let is_spent = electrs.script_get_history(&txout_script).unwrap().len() >= 2; if is_spent { return; } @@ -577,12 +635,30 @@ pub(crate) async fn wait_for_outpoint_spend(electrs: &E, outpoin exponential_backoff_poll(|| { electrs.ping().unwrap(); - let is_spent = !electrs.script_get_history(&txout_script).unwrap().is_empty(); + let is_spent = electrs.script_get_history(&txout_script).unwrap().len() >= 2; is_spent.then_some(()) }) .await; } +pub(crate) async fn wait_for_cbf_sync(node: &TestNode) { + let before = node.status().latest_onchain_wallet_sync_timestamp; + let mut delay = Duration::from_millis(200); + for _ in 0..30 { + if node.sync_wallets().is_ok() { + let after = node.status().latest_onchain_wallet_sync_timestamp; + if after > before { + return; + } + } + tokio::time::sleep(delay).await; + if delay < Duration::from_secs(2) { + delay = delay.mul_f32(1.5); + } + } + panic!("wait_for_cbf_sync: timed out waiting for CBF sync to complete"); +} + pub(crate) async fn exponential_backoff_poll(mut poll: F) -> T where F: FnMut() -> Option, @@ -1221,8 +1297,9 @@ pub(crate) async fn do_channel_full_cycle( let splice_out_sat = funding_amount_sat / 2; node_b.splice_out(&user_channel_id_b, node_a.node_id(), &addr_a, splice_out_sat).unwrap(); - expect_splice_pending_event!(node_a, node_b.node_id()); + let splice_out_txo = expect_splice_pending_event!(node_a, node_b.node_id()); expect_splice_pending_event!(node_b, node_a.node_id()); + wait_for_tx(electrsd, splice_out_txo.txid).await; generate_blocks_and_wait(&bitcoind, electrsd, 6).await; node_a.sync_wallets().unwrap(); @@ -1243,8 +1320,9 @@ pub(crate) async fn do_channel_full_cycle( let splice_in_sat = splice_out_sat; node_a.splice_in(&user_channel_id_a, node_b.node_id(), splice_in_sat).unwrap(); - expect_splice_pending_event!(node_a, node_b.node_id()); + let splice_in_txo = expect_splice_pending_event!(node_a, node_b.node_id()); expect_splice_pending_event!(node_b, node_a.node_id()); + wait_for_tx(electrsd, splice_in_txo.txid).await; generate_blocks_and_wait(&bitcoind, electrsd, 6).await; node_a.sync_wallets().unwrap(); @@ -1272,12 +1350,22 @@ pub(crate) async fn do_channel_full_cycle( expect_event!(node_a, ChannelClosed); expect_event!(node_b, ChannelClosed); - wait_for_outpoint_spend(electrsd, funding_txo_b).await; + // After splices, the latest funding outpoint is from the last splice. + // We must wait for the close tx (which spends the latest funding output) + // to propagate before mining. + wait_for_outpoint_spend(electrsd, splice_in_txo).await; generate_blocks_and_wait(&bitcoind, electrsd, 1).await; node_a.sync_wallets().unwrap(); node_b.sync_wallets().unwrap(); + // CBF needs a second sync: the first sync confirms the close tx in the + // Lightning wallet, which may trigger new script registrations. The + // second sync picks up blocks matching those new scripts for the + // on-chain wallet. + node_a.sync_wallets().unwrap(); + node_b.sync_wallets().unwrap(); + if force_close { // Check node_b properly sees all balances and sweeps them. assert_eq!(node_b.list_balances().lightning_balances.len(), 1); diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index 413b2d44ab..94805de1a2 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -23,8 +23,9 @@ use common::{ expect_payment_successful_event, expect_splice_pending_event, generate_blocks_and_wait, generate_listening_addresses, open_channel, open_channel_push_amt, open_channel_with_all, premine_and_distribute_funds, premine_blocks, prepare_rbf, random_chain_source, random_config, - setup_bitcoind_and_electrsd, setup_builder, setup_node, setup_two_nodes, splice_in_with_all, - wait_for_tx, TestChainSource, TestStoreType, TestSyncStore, + setup_bitcoind_and_electrsd, setup_builder, setup_node, setup_two_nodes, skip_if_cbf, + splice_in_with_all, wait_for_cbf_sync, wait_for_tx, TestChainSource, TestStoreType, + TestSyncStore, }; use electrsd::corepc_node::Node as BitcoinD; use electrsd::ElectrsD; @@ -74,6 +75,7 @@ async fn channel_full_cycle_force_close_trusted_no_reserve() { async fn channel_full_cycle_0conf() { let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); let chain_source = random_chain_source(&bitcoind, &electrsd); + skip_if_cbf!(chain_source); let (node_a, node_b) = setup_two_nodes(&chain_source, true, true, false); do_channel_full_cycle(node_a, node_b, &bitcoind.client, &electrsd.client, true, true, false) .await; @@ -977,6 +979,7 @@ async fn splice_channel() { let txo = expect_splice_pending_event!(node_a, node_b.node_id()); expect_splice_pending_event!(node_b, node_a.node_id()); + wait_for_tx(&electrsd.client, txo.txid).await; generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 6).await; @@ -1020,6 +1023,7 @@ async fn splice_channel() { let txo = expect_splice_pending_event!(node_a, node_b.node_id()); expect_splice_pending_event!(node_b, node_a.node_id()); + wait_for_tx(&electrsd.client, txo.txid).await; generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 6).await; @@ -1668,8 +1672,8 @@ async fn unified_send_receive_bip21_uri() { }, }; - generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 6).await; wait_for_tx(&electrsd.client, txid).await; + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 6).await; node_a.sync_wallets().unwrap(); node_b.sync_wallets().unwrap(); @@ -2554,6 +2558,7 @@ async fn persistence_backwards_compatibility() { async fn onchain_fee_bump_rbf() { let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); let chain_source = random_chain_source(&bitcoind, &electrsd); + skip_if_cbf!(chain_source); let (node_a, node_b) = setup_two_nodes(&chain_source, false, true, false); // Fund both nodes @@ -2873,3 +2878,336 @@ async fn splice_in_with_all_balance() { node_a.stop().unwrap(); node_b.stop().unwrap(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn start_stop_cbf() { + let (bitcoind, _electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = TestChainSource::Cbf(&bitcoind); + let node = setup_node(&chain_source, random_config(true)); + + assert!(node.status().is_running); + node.stop().unwrap(); + assert_eq!(node.stop(), Err(NodeError::NotRunning)); + + node.start().unwrap(); + assert_eq!(node.start(), Err(NodeError::AlreadyRunning)); + assert!(node.status().is_running); + + node.stop().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn fee_rate_estimation_after_manual_sync_cbf() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = TestChainSource::Cbf(&bitcoind); + let node = setup_node(&chain_source, random_config(true)); + + let addr = node.onchain_payment().new_address().unwrap(); + premine_and_distribute_funds( + &bitcoind.client, + &electrsd.client, + vec![addr], + Amount::from_sat(100_000), + ) + .await; + + wait_for_cbf_sync(&node).await; + let first_fee_update = node.status().latest_fee_rate_cache_update_timestamp; + assert!(first_fee_update.is_some()); + + // Subsequent manual syncs should keep the fee cache populated. + node.sync_wallets().unwrap(); + let second_fee_update = node.status().latest_fee_rate_cache_update_timestamp; + assert!(second_fee_update.is_some()); + assert!(second_fee_update >= first_fee_update); + + node.stop().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn repeated_manual_sync_cbf() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = TestChainSource::Cbf(&bitcoind); + let node = setup_node(&chain_source, random_config(true)); + + let addr = node.onchain_payment().new_address().unwrap(); + let premine_amount_sat = 100_000; + + premine_and_distribute_funds( + &bitcoind.client, + &electrsd.client, + vec![addr], + Amount::from_sat(premine_amount_sat), + ) + .await; + + wait_for_cbf_sync(&node).await; + assert_eq!(node.list_balances().spendable_onchain_balance_sats, premine_amount_sat); + + // Regression: the second manual sync must not block forever. + node.sync_wallets().unwrap(); + assert_eq!(node.list_balances().spendable_onchain_balance_sats, premine_amount_sat); + + node.stop().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn start_stop_reinit_cbf() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let config = random_config(true); + + let p2p_socket = bitcoind.params.p2p_socket.expect("P2P must be enabled for CBF"); + let peer_addr = format!("{}", p2p_socket); + let sync_config = + ldk_node::config::CbfSyncConfig { background_sync_config: None, ..Default::default() }; + + let test_sync_store = TestSyncStore::new(config.node_config.storage_dir_path.clone().into()); + + setup_builder!(builder, config.node_config); + builder.set_chain_source_cbf(vec![peer_addr.clone()], Some(sync_config.clone()), None); + + let node = builder + .build_with_store(config.node_entropy.clone().into(), test_sync_store.clone()) + .unwrap(); + node.start().unwrap(); + + let expected_node_id = node.node_id(); + assert_eq!(node.start(), Err(NodeError::AlreadyRunning)); + + let funding_address = node.onchain_payment().new_address().unwrap(); + assert_eq!(node.list_balances().total_onchain_balance_sats, 0); + + let expected_amount = Amount::from_sat(100_000); + premine_and_distribute_funds( + &bitcoind.client, + &electrsd.client, + vec![funding_address], + expected_amount, + ) + .await; + + wait_for_cbf_sync(&node).await; + assert_eq!(node.list_balances().spendable_onchain_balance_sats, expected_amount.to_sat()); + + node.stop().unwrap(); + assert_eq!(node.stop(), Err(NodeError::NotRunning)); + + node.start().unwrap(); + assert_eq!(node.start(), Err(NodeError::AlreadyRunning)); + + node.stop().unwrap(); + assert_eq!(node.stop(), Err(NodeError::NotRunning)); + drop(node); + + // Reinitialize from the same config and store. + setup_builder!(builder, config.node_config); + builder.set_chain_source_cbf(vec![peer_addr], Some(sync_config), None); + + let reinitialized_node = + builder.build_with_store(config.node_entropy.into(), test_sync_store).unwrap(); + reinitialized_node.start().unwrap(); + assert_eq!(reinitialized_node.node_id(), expected_node_id); + + // Balance should be persisted from the previous run. + assert_eq!( + reinitialized_node.list_balances().spendable_onchain_balance_sats, + expected_amount.to_sat() + ); + + wait_for_cbf_sync(&reinitialized_node).await; + assert_eq!( + reinitialized_node.list_balances().spendable_onchain_balance_sats, + expected_amount.to_sat() + ); + + reinitialized_node.stop().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn onchain_wallet_recovery_cbf() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = TestChainSource::Cbf(&bitcoind); + + let original_config = random_config(true); + let original_node_entropy = original_config.node_entropy.clone(); + let original_node = setup_node(&chain_source, original_config); + + let premine_amount_sat = 100_000; + + let addr_1 = original_node.onchain_payment().new_address().unwrap(); + + premine_and_distribute_funds( + &bitcoind.client, + &electrsd.client, + vec![addr_1], + Amount::from_sat(premine_amount_sat), + ) + .await; + + wait_for_cbf_sync(&original_node).await; + assert_eq!(original_node.list_balances().spendable_onchain_balance_sats, premine_amount_sat); + + let addr_2 = original_node.onchain_payment().new_address().unwrap(); + + let txid = bitcoind + .client + .send_to_address(&addr_2, Amount::from_sat(premine_amount_sat)) + .unwrap() + .0 + .parse() + .unwrap(); + wait_for_tx(&electrsd.client, txid).await; + + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 1).await; + + wait_for_cbf_sync(&original_node).await; + assert_eq!( + original_node.list_balances().spendable_onchain_balance_sats, + premine_amount_sat * 2 + ); + + original_node.stop().unwrap(); + drop(original_node); + + // Now we start from scratch, only the seed remains the same. + let mut recovered_config = random_config(true); + recovered_config.node_entropy = original_node_entropy; + recovered_config.recovery_mode = true; + let recovered_node = setup_node(&chain_source, recovered_config); + + wait_for_cbf_sync(&recovered_node).await; + assert_eq!( + recovered_node.list_balances().spendable_onchain_balance_sats, + premine_amount_sat * 2 + ); + + // Check we sync even when skipping some addresses. + let _addr_3 = recovered_node.onchain_payment().new_address().unwrap(); + let _addr_4 = recovered_node.onchain_payment().new_address().unwrap(); + let _addr_5 = recovered_node.onchain_payment().new_address().unwrap(); + let addr_6 = recovered_node.onchain_payment().new_address().unwrap(); + + let txid = bitcoind + .client + .send_to_address(&addr_6, Amount::from_sat(premine_amount_sat)) + .unwrap() + .0 + .parse() + .unwrap(); + wait_for_tx(&electrsd.client, txid).await; + + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 1).await; + + wait_for_cbf_sync(&recovered_node).await; + assert_eq!( + recovered_node.list_balances().spendable_onchain_balance_sats, + premine_amount_sat * 3 + ); + + recovered_node.stop().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn onchain_send_receive_cbf() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = TestChainSource::Cbf(&bitcoind); + let (node_a, node_b) = setup_two_nodes(&chain_source, false, true, false); + + let addr_a = node_a.onchain_payment().new_address().unwrap(); + let addr_b = node_b.onchain_payment().new_address().unwrap(); + + let premine_amount_sat = 1_100_000; + premine_and_distribute_funds( + &bitcoind.client, + &electrsd.client, + vec![addr_a.clone(), addr_b.clone()], + Amount::from_sat(premine_amount_sat), + ) + .await; + + wait_for_cbf_sync(&node_a).await; + node_b.sync_wallets().unwrap(); + assert_eq!(node_a.list_balances().spendable_onchain_balance_sats, premine_amount_sat); + assert_eq!(node_b.list_balances().spendable_onchain_balance_sats, premine_amount_sat); + + // Check on-chain payment tracking after premine. + let node_a_payments = node_a.list_payments(); + let node_b_payments = node_b.list_payments(); + for payments in [&node_a_payments, &node_b_payments] { + assert_eq!(payments.len(), 1); + } + for p in [node_a_payments.first().unwrap(), node_b_payments.first().unwrap()] { + assert_eq!(p.amount_msat, Some(premine_amount_sat * 1000)); + assert_eq!(p.direction, PaymentDirection::Inbound); + assert_eq!(p.status, PaymentStatus::Pending); + match p.kind { + PaymentKind::Onchain { status, .. } => { + assert!(matches!(status, ConfirmationStatus::Confirmed { .. })); + }, + _ => panic!("Unexpected payment kind"), + } + } + + // Send from B to A. + let amount_to_send_sats = 54_321; + let txid = + node_b.onchain_payment().send_to_address(&addr_a, amount_to_send_sats, None).unwrap(); + wait_for_tx(&electrsd.client, txid).await; + + // Mine the transaction so CBF can see it. + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 6).await; + wait_for_cbf_sync(&node_a).await; + node_b.sync_wallets().unwrap(); + + let payment_id = PaymentId(txid.to_byte_array()); + let payment_a = node_a.payment(&payment_id).unwrap(); + match payment_a.kind { + PaymentKind::Onchain { txid: tx, status } => { + assert_eq!(tx, txid); + assert!(matches!(status, ConfirmationStatus::Confirmed { .. })); + }, + _ => panic!("Unexpected payment kind"), + } + assert!(payment_a.fee_paid_msat > Some(0)); + assert_eq!(payment_a.amount_msat, Some(amount_to_send_sats * 1000)); + + let payment_b = node_b.payment(&payment_id).unwrap(); + match payment_b.kind { + PaymentKind::Onchain { txid: tx, status } => { + assert_eq!(tx, txid); + assert!(matches!(status, ConfirmationStatus::Confirmed { .. })); + }, + _ => panic!("Unexpected payment kind"), + } + assert!(payment_b.fee_paid_msat > Some(0)); + assert_eq!(payment_b.amount_msat, Some(amount_to_send_sats * 1000)); + assert_eq!(payment_a.fee_paid_msat, payment_b.fee_paid_msat); + + let onchain_fee_buffer_sat = 1000; + let expected_node_a_balance = premine_amount_sat + amount_to_send_sats; + assert_eq!(node_a.list_balances().spendable_onchain_balance_sats, expected_node_a_balance); + assert!( + node_b.list_balances().spendable_onchain_balance_sats + > premine_amount_sat - amount_to_send_sats - onchain_fee_buffer_sat + ); + assert!( + node_b.list_balances().spendable_onchain_balance_sats + < premine_amount_sat - amount_to_send_sats + ); + + // Test send_all_to_address. + let addr_b2 = node_b.onchain_payment().new_address().unwrap(); + let txid = node_a.onchain_payment().send_all_to_address(&addr_b2, false, None).unwrap(); + wait_for_tx(&electrsd.client, txid).await; + + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 6).await; + wait_for_cbf_sync(&node_a).await; + node_b.sync_wallets().unwrap(); + + assert_eq!(node_a.list_balances().spendable_onchain_balance_sats, 0); + assert_eq!(node_a.list_balances().total_onchain_balance_sats, 0); + assert!(node_b.list_balances().spendable_onchain_balance_sats > premine_amount_sat); + + node_a.stop().unwrap(); + node_b.stop().unwrap(); +} From 3776ad36daf4589a715c0ce395cd0f9dddd526ac Mon Sep 17 00:00:00 2001 From: Yeji Han Date: Sat, 4 Apr 2026 02:04:54 +0900 Subject: [PATCH 11/15] refactor(cbf): extract build_cbf_node helper and add wallet reference Preparation for auto-restart: extract bip157 node build logic into a reusable helper method, add chain_state() from wallet checkpoint to avoid genesis re-sync, and thread Arc through start(). AI: claude --- src/chain/cbf.rs | 71 ++++++++++++++++++++++++++++++++++++++++-------- src/chain/mod.rs | 6 ++-- src/lib.rs | 10 ++++--- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/chain/cbf.rs b/src/chain/cbf.rs index 0c76401967..693ea23b19 100644 --- a/src/chain/cbf.rs +++ b/src/chain/cbf.rs @@ -13,9 +13,10 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bdk_chain::{BlockId, ConfirmationBlockTime, TxUpdate}; use bdk_wallet::Update; -use bip157::chain::BlockHeaderChanges; +use bip157::chain::{BlockHeaderChanges, ChainState}; use bip157::{ - BlockHash, Builder, Client, Event, Info, Requester, SyncUpdate, TrustedPeer, Warning, + BlockHash, Builder, Client, Event, HeaderCheckpoint, Info, Node as CbfNode, Requester, + SyncUpdate, TrustedPeer, Warning, }; use bitcoin::constants::SUBSIDY_HALVING_INTERVAL; use bitcoin::{Amount, FeeRate, Network, Script, ScriptBuf, Transaction, Txid}; @@ -48,6 +49,12 @@ const FEE_RATE_LOOKBACK_BLOCKS: usize = 6; /// Matches bdk-kyoto's `IMPOSSIBLE_REORG_DEPTH`. const REORG_SAFETY_BLOCKS: u32 = 7; +/// Maximum consecutive restart attempts before giving up. +const MAX_RESTART_RETRIES: u32 = 5; + +/// Initial backoff delay for restart retries (doubles each attempt). +const INITIAL_BACKOFF_MS: u64 = 500; + /// The fee estimation back-end used by the CBF chain source. enum FeeSource { /// Derive fee rates from the coinbase reward of recent blocks. @@ -97,6 +104,8 @@ pub(super) struct CbfChainSource { kv_store: Arc, /// Node configuration (network, storage path, etc.). config: Arc, + /// On-chain wallet reference for deriving chain_state checkpoints on restart. + onchain_wallet: Mutex>>, /// Logger instance. logger: Arc, /// Shared node metrics (sync timestamps, etc.). @@ -148,6 +157,7 @@ impl CbfChainSource { let scan_lock = tokio::sync::Mutex::new(()); let onchain_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); let lightning_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); + let onchain_wallet = Mutex::new(None); Ok(Self { peers, sync_config, @@ -163,6 +173,7 @@ impl CbfChainSource { onchain_wallet_sync_status, lightning_wallet_sync_status, fee_estimator, + onchain_wallet, kv_store, config, logger, @@ -170,14 +181,13 @@ impl CbfChainSource { }) } - /// Start the bip157 node and spawn background tasks for event processing. - pub(crate) fn start(&self, runtime: Arc) { - let mut status = self.cbf_runtime_status.lock().unwrap(); - if matches!(*status, CbfRuntimeStatus::Started { .. }) { - debug_assert!(false, "We shouldn't call start if we're already started"); - return; - } - + /// Build a new bip157 node and client from the current configuration. + /// + /// If an on-chain wallet reference is available, a `ChainState::Checkpoint` + /// is derived from the wallet's persisted checkpoint (walked back by + /// `REORG_SAFETY_BLOCKS`) so the node resumes near its last known height + /// instead of re-syncing from genesis. + fn build_cbf_node(&self) -> (CbfNode, Client) { let network = self.config.network; let mut builder = Builder::new(network); @@ -210,7 +220,46 @@ impl CbfChainSource { builder = builder.response_timeout(Duration::from_secs(self.sync_config.response_timeout_secs)); - let (node, client) = builder.build(); + // If we have a wallet reference, derive a chain_state checkpoint so the + // bip157 node can skip already-synced headers on restart. + if let Some(wallet) = self.onchain_wallet.lock().unwrap().as_ref() { + let cp = wallet.latest_checkpoint(); + let target_height = cp.height().saturating_sub(REORG_SAFETY_BLOCKS); + // Walk the checkpoint chain back to the target height. + let mut cursor = cp; + while cursor.height() > target_height { + match cursor.prev() { + Some(prev) => cursor = prev, + None => break, + } + } + if cursor.height() > 0 { + let header_cp = HeaderCheckpoint::new(cursor.height(), cursor.hash()); + builder = builder.chain_state(ChainState::Checkpoint(header_cp)); + log_debug!( + self.logger, + "CBF builder: resuming from checkpoint height={}, hash={}", + cursor.height(), + cursor.hash(), + ); + } + } + + builder.build() + } + + /// Start the bip157 node and spawn background tasks for event processing. + pub(crate) fn start(&self, runtime: Arc, onchain_wallet: Arc) { + let mut status = self.cbf_runtime_status.lock().unwrap(); + if matches!(*status, CbfRuntimeStatus::Started { .. }) { + debug_assert!(false, "We shouldn't call start if we're already started"); + return; + } + + // Store the wallet reference for future restarts. + *self.onchain_wallet.lock().unwrap() = Some(Arc::clone(&onchain_wallet)); + + let (node, client) = self.build_cbf_node(); let Client { requester, info_rx, warn_rx, event_rx } = client; diff --git a/src/chain/mod.rs b/src/chain/mod.rs index b896ba6fbf..cf544e571d 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -222,13 +222,15 @@ impl ChainSource { Ok((Self { kind, registered_txids, tx_broadcaster, logger }, None)) } - pub(crate) fn start(&self, runtime: Arc) -> Result<(), Error> { + pub(crate) fn start( + &self, runtime: Arc, onchain_wallet: Arc, + ) -> Result<(), Error> { match &self.kind { ChainSourceKind::Electrum(electrum_chain_source) => { electrum_chain_source.start(runtime)? }, ChainSourceKind::Cbf(cbf_chain_source) => { - cbf_chain_source.start(runtime); + cbf_chain_source.start(runtime, onchain_wallet); }, _ => { // Nothing to do for other chain sources. diff --git a/src/lib.rs b/src/lib.rs index 1f6df49fc2..326e197a04 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -268,10 +268,12 @@ impl Node { ); // Start up any runtime-dependant chain sources (e.g. Electrum) - self.chain_source.start(Arc::clone(&self.runtime)).map_err(|e| { - log_error!(self.logger, "Failed to start chain syncing: {}", e); - e - })?; + self.chain_source.start(Arc::clone(&self.runtime), Arc::clone(&self.wallet)).map_err( + |e| { + log_error!(self.logger, "Failed to start chain syncing: {}", e); + e + }, + )?; // Block to ensure we update our fee rate cache once on startup let chain_source = Arc::clone(&self.chain_source); From dd951a7ea6698c941230e262519cadc5ec8ab212 Mon Sep 17 00:00:00 2001 From: Yeji Han Date: Sat, 4 Apr 2026 02:35:12 +0900 Subject: [PATCH 12/15] feat(cbf): auto-restart bip157 node with exponential backoff When node.run() exits (e.g. NoReachablePeers from kyoto #558), the background task rebuilds the node, swaps the requester, and respawns channel processing tasks, up to MAX_RESTART_RETRIES (5) attempts with doubling backoff starting at 500ms. - Change cbf_runtime_status from Mutex<> to Arc> so it can be shared with the async restart loop - Extract build_cbf_node_static() that takes explicit params instead of &self, enabling calls from 'static async blocks - Move all task spawning (info/warn/event + node.run) into a single restart loop inside spawn_background_task AI: claude --- src/chain/cbf.rs | 192 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 142 insertions(+), 50 deletions(-) diff --git a/src/chain/cbf.rs b/src/chain/cbf.rs index 693ea23b19..44611466ab 100644 --- a/src/chain/cbf.rs +++ b/src/chain/cbf.rs @@ -79,7 +79,7 @@ pub(super) struct CbfChainSource { /// Fee estimation back-end. fee_source: FeeSource, /// Tracks whether the bip157 node is running and holds the command handle. - cbf_runtime_status: Mutex, + cbf_runtime_status: Arc>, /// Latest chain tip hash, updated by the background event processing task. latest_tip: Arc>>, /// Scripts to match against compact block filters during a scan. @@ -147,7 +147,7 @@ impl CbfChainSource { None => FeeSource::Cbf, }; - let cbf_runtime_status = Mutex::new(CbfRuntimeStatus::Stopped); + let cbf_runtime_status = Arc::new(Mutex::new(CbfRuntimeStatus::Stopped)); let latest_tip = Arc::new(Mutex::new(None)); let watched_scripts = Arc::new(RwLock::new(Vec::new())); let matched_block_hashes = Arc::new(Mutex::new(Vec::new())); @@ -183,46 +183,57 @@ impl CbfChainSource { /// Build a new bip157 node and client from the current configuration. /// - /// If an on-chain wallet reference is available, a `ChainState::Checkpoint` - /// is derived from the wallet's persisted checkpoint (walked back by - /// `REORG_SAFETY_BLOCKS`) so the node resumes near its last known height - /// instead of re-syncing from genesis. + /// Delegates to [`Self::build_cbf_node_static`], passing all needed fields. fn build_cbf_node(&self) -> (CbfNode, Client) { - let network = self.config.network; + let wallet = self.onchain_wallet.lock().unwrap().clone(); + Self::build_cbf_node_static( + &self.peers, + &self.sync_config, + &self.config, + wallet.as_ref(), + &self.logger, + ) + } + + /// Static version of the builder: takes all required parameters explicitly + /// so it can be called from an `async move` block without borrowing `self`. + fn build_cbf_node_static( + peers: &[String], sync_config: &CbfSyncConfig, config: &Config, + wallet: Option<&Arc>, logger: &Logger, + ) -> (CbfNode, Client) { + let network = config.network; let mut builder = Builder::new(network); // Configure data directory under the node's storage path. - let data_dir = std::path::PathBuf::from(&self.config.storage_dir_path).join("bip157_data"); + let data_dir = std::path::PathBuf::from(&config.storage_dir_path).join("bip157_data"); builder = builder.data_dir(data_dir); // Add configured peers. - let peers: Vec = self - .peers + let trusted_peers: Vec = peers .iter() .filter_map(|peer_str| { peer_str.parse::().ok().map(TrustedPeer::from_socket_addr) }) .collect(); - if !peers.is_empty() { - builder = builder.add_peers(peers); + if !trusted_peers.is_empty() { + builder = builder.add_peers(trusted_peers); } // Require multiple peers to agree on filter headers before accepting them, // as recommended by BIP 157 to mitigate malicious peer attacks. - builder = builder.required_peers(self.sync_config.required_peers); + builder = builder.required_peers(sync_config.required_peers); // Request witness data so segwit transactions include full witnesses, // required for Lightning channel operations. builder = builder.fetch_witness_data(); // Set peer response timeout from user configuration (default: 30s). - builder = - builder.response_timeout(Duration::from_secs(self.sync_config.response_timeout_secs)); + builder = builder.response_timeout(Duration::from_secs(sync_config.response_timeout_secs)); // If we have a wallet reference, derive a chain_state checkpoint so the // bip157 node can skip already-synced headers on restart. - if let Some(wallet) = self.onchain_wallet.lock().unwrap().as_ref() { + if let Some(wallet) = wallet { let cp = wallet.latest_checkpoint(); let target_height = cp.height().saturating_sub(REORG_SAFETY_BLOCKS); // Walk the checkpoint chain back to the target height. @@ -237,7 +248,7 @@ impl CbfChainSource { let header_cp = HeaderCheckpoint::new(cursor.height(), cursor.hash()); builder = builder.chain_state(ChainState::Checkpoint(header_cp)); log_debug!( - self.logger, + logger, "CBF builder: resuming from checkpoint height={}, hash={}", cursor.height(), cursor.hash(), @@ -249,6 +260,11 @@ impl CbfChainSource { } /// Start the bip157 node and spawn background tasks for event processing. + /// + /// The node runs inside a restart loop: if `node.run()` returns an error, + /// the loop rebuilds the node, swaps the requester, and respawns channel + /// processing tasks — up to [`MAX_RESTART_RETRIES`] consecutive failures + /// with exponential backoff starting at [`INITIAL_BACKOFF_MS`]. pub(crate) fn start(&self, runtime: Arc, onchain_wallet: Arc) { let mut status = self.cbf_runtime_status.lock().unwrap(); if matches!(*status, CbfRuntimeStatus::Started { .. }) { @@ -263,42 +279,118 @@ impl CbfChainSource { let Client { requester, info_rx, warn_rx, event_rx } = client; - // Spawn the bip157 node in the background. - let node_logger = Arc::clone(&self.logger); - runtime.spawn_background_task(async move { - if let Err(e) = node.run().await { - log_error!(node_logger, "CBF node exited with error: {:?}", e); - } - }); - - // Spawn a task to log info messages. - let info_logger = Arc::clone(&self.logger); - runtime - .spawn_cancellable_background_task(Self::process_info_messages(info_rx, info_logger)); - - // Spawn a task to log warning messages. - let warn_logger = Arc::clone(&self.logger); - runtime - .spawn_cancellable_background_task(Self::process_warn_messages(warn_rx, warn_logger)); - - // Spawn a task to process events. - let event_state = CbfEventState { - latest_tip: Arc::clone(&self.latest_tip), - watched_scripts: Arc::clone(&self.watched_scripts), - matched_block_hashes: Arc::clone(&self.matched_block_hashes), - sync_completion_tx: Arc::clone(&self.sync_completion_tx), - filter_skip_height: Arc::clone(&self.filter_skip_height), - }; - let event_logger = Arc::clone(&self.logger); - runtime.spawn_cancellable_background_task(Self::process_events( - event_rx, - event_state, - event_logger, - )); + *status = CbfRuntimeStatus::Started { requester }; + drop(status); log_info!(self.logger, "CBF chain source started."); - *status = CbfRuntimeStatus::Started { requester }; + // Clone all Arc references needed by the restart loop so the async + // block is 'static (no borrows of `self`). + let restart_status = Arc::clone(&self.cbf_runtime_status); + let restart_logger = Arc::clone(&self.logger); + let restart_latest_tip = Arc::clone(&self.latest_tip); + let restart_watched_scripts = Arc::clone(&self.watched_scripts); + let restart_matched_block_hashes = Arc::clone(&self.matched_block_hashes); + let restart_sync_completion_tx = Arc::clone(&self.sync_completion_tx); + let restart_filter_skip_height = Arc::clone(&self.filter_skip_height); + let restart_peers = self.peers.clone(); + let restart_sync_config = self.sync_config.clone(); + let restart_config = Arc::clone(&self.config); + let restart_wallet = Arc::clone(&onchain_wallet); + + runtime.spawn_background_task(async move { + let mut current_node = node; + let mut current_info_rx = info_rx; + let mut current_warn_rx = warn_rx; + let mut current_event_rx = event_rx; + let mut retries = 0u32; + let mut backoff_ms = INITIAL_BACKOFF_MS; + + loop { + // Spawn channel processing tasks for this iteration. + let info_handle = tokio::spawn(Self::process_info_messages( + current_info_rx, + Arc::clone(&restart_logger), + )); + let warn_handle = tokio::spawn(Self::process_warn_messages( + current_warn_rx, + Arc::clone(&restart_logger), + )); + let event_state = CbfEventState { + latest_tip: Arc::clone(&restart_latest_tip), + watched_scripts: Arc::clone(&restart_watched_scripts), + matched_block_hashes: Arc::clone(&restart_matched_block_hashes), + sync_completion_tx: Arc::clone(&restart_sync_completion_tx), + filter_skip_height: Arc::clone(&restart_filter_skip_height), + }; + let event_handle = tokio::spawn(Self::process_events( + current_event_rx, + event_state, + Arc::clone(&restart_logger), + )); + + // Run the node until it exits. + match current_node.run().await { + Ok(()) => { + log_info!(restart_logger, "CBF node shut down cleanly."); + break; + }, + Err(e) => { + retries += 1; + if retries > MAX_RESTART_RETRIES { + log_error!( + restart_logger, + "CBF node failed {} times, giving up: {:?}", + retries, + e, + ); + break; + } + log_error!( + restart_logger, + "CBF node exited with error (attempt {}/{}): {:?}. \ + Restarting in {}ms.", + retries, + MAX_RESTART_RETRIES, + e, + backoff_ms, + ); + + tokio::time::sleep(Duration::from_millis(backoff_ms)).await; + backoff_ms = backoff_ms.saturating_mul(2); + + // Abort old channel processing tasks. + info_handle.abort(); + warn_handle.abort(); + event_handle.abort(); + + // Rebuild the node from scratch. + let (new_node, new_client) = Self::build_cbf_node_static( + &restart_peers, + &restart_sync_config, + &restart_config, + Some(&restart_wallet), + &restart_logger, + ); + let Client { + requester: new_requester, + info_rx: new_info_rx, + warn_rx: new_warn_rx, + event_rx: new_event_rx, + } = new_client; + + // Swap the requester so callers pick up the new handle. + *restart_status.lock().unwrap() = + CbfRuntimeStatus::Started { requester: new_requester }; + + current_node = new_node; + current_info_rx = new_info_rx; + current_warn_rx = new_warn_rx; + current_event_rx = new_event_rx; + }, + } + } + }); } /// Shut down the bip157 node and stop all background tasks. From aac606181a3986ffd7e0a0ffa8e70c17a6a6af9a Mon Sep 17 00:00:00 2001 From: Yeji Han Date: Sat, 4 Apr 2026 02:50:37 +0900 Subject: [PATCH 13/15] feat(cbf): add liveness check before returning requester requester() now checks is_running() to give callers an immediate failure signal instead of waiting for SendError to propagate through the channel. --- src/chain/cbf.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/chain/cbf.rs b/src/chain/cbf.rs index 44611466ab..6a0cf13220 100644 --- a/src/chain/cbf.rs +++ b/src/chain/cbf.rs @@ -481,7 +481,16 @@ impl CbfChainSource { fn requester(&self) -> Result { let status = self.cbf_runtime_status.lock().unwrap(); match &*status { - CbfRuntimeStatus::Started { requester } => Ok(requester.clone()), + CbfRuntimeStatus::Started { requester } if requester.is_running() => { + Ok(requester.clone()) + }, + CbfRuntimeStatus::Started { .. } => { + log_error!( + self.logger, + "CBF node is not running; sync will fail until restart completes." + ); + Err(Error::ConnectionFailed) + }, CbfRuntimeStatus::Stopped => { debug_assert!( false, From 8e097fb89decb3b03ef6f489ee2eca21dc0e525a Mon Sep 17 00:00:00 2001 From: Yeji Han Date: Sat, 4 Apr 2026 02:53:02 +0900 Subject: [PATCH 14/15] fix(cbf): clean up scan state on filter scan failure Extract cleanup_scan_state() helper and call it on error paths in run_filter_scan() to prevent stale watched_scripts, matched_block_hashes, and filter_skip_height from leaking between scans. --- src/chain/cbf.rs | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/chain/cbf.rs b/src/chain/cbf.rs index 6a0cf13220..d8a8d1832a 100644 --- a/src/chain/cbf.rs +++ b/src/chain/cbf.rs @@ -501,6 +501,19 @@ impl CbfChainSource { } } + /// Reset filter scan state to a clean baseline. + /// + /// Called on both success and error paths in `run_filter_scan()` to ensure + /// no stale state leaks between scans. + fn cleanup_scan_state(&self) { + self.filter_skip_height.store(0, Ordering::Release); + self.watched_scripts.write().unwrap().clear(); + self.matched_block_hashes.lock().unwrap().clear(); + if let Some(tx) = self.sync_completion_tx.lock().unwrap().take() { + drop(tx); + } + } + /// Register a transaction script for Lightning channel monitoring. pub(crate) fn register_tx(&self, _txid: &Txid, script_pubkey: &Script) { self.registered_scripts.lock().unwrap().push(script_pubkey.to_owned()); @@ -530,21 +543,27 @@ impl CbfChainSource { let (tx, rx) = oneshot::channel(); *self.sync_completion_tx.lock().unwrap() = Some(tx); - requester.rescan().map_err(|e| { + if let Err(e) = requester.rescan().map_err(|e| { log_error!(self.logger, "Failed to trigger CBF rescan: {:?}", e); Error::WalletOperationFailed - })?; - - let sync_update = rx.await.map_err(|e| { - log_error!(self.logger, "CBF sync completion channel dropped: {:?}", e); - Error::WalletOperationFailed - })?; - - self.filter_skip_height.store(0, Ordering::Release); - self.watched_scripts.write().unwrap().clear(); - let matched = std::mem::take(&mut *self.matched_block_hashes.lock().unwrap()); + }) { + self.cleanup_scan_state(); + return Err(e); + } - Ok((sync_update, matched)) + match rx.await { + Ok(sync_update) => { + self.filter_skip_height.store(0, Ordering::Release); + self.watched_scripts.write().unwrap().clear(); + let matched = std::mem::take(&mut *self.matched_block_hashes.lock().unwrap()); + Ok((sync_update, matched)) + }, + Err(e) => { + log_error!(self.logger, "CBF sync completion channel dropped: {:?}", e); + self.cleanup_scan_state(); + Err(Error::WalletOperationFailed) + }, + } } /// Sync the on-chain wallet by scanning compact block filters for relevant transactions. From a138a47fb60df5d5fe7e3a11151d13b7ab5e5bac Mon Sep 17 00:00:00 2001 From: Yeji Han Date: Sat, 4 Apr 2026 02:57:09 +0900 Subject: [PATCH 15/15] fix(cbf): add per-block-fetch timeout to wallet sync Wrap individual get_block() calls in tokio::time::timeout using the existing per_request_timeout_secs config. Previously only the overall sync had a timeout; individual block fetches could hang indefinitely (kyoto #556). --- .github/workflows/rust.yml | 4 ++++ src/chain/cbf.rs | 41 ++++++++++++++++++++++++++++---------- src/config.rs | 4 ++-- tests/common/mod.rs | 2 +- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fcda2c83e1..2cb29d5a30 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -41,6 +41,10 @@ jobs: - name: Check formatting on Rust ${{ matrix.toolchain }} if: matrix.check-fmt run: rustup component add rustfmt && cargo fmt --all -- --check + - name: Pin packages to allow for MSRV + if: matrix.msrv + run: | + cargo update -p idna_adapter --precise "1.2.0" --verbose # idna_adapter 1.2.1 uses ICU4X 2.2.0, requiring 1.86 and newer - name: Set RUSTFLAGS to deny warnings if: "matrix.toolchain == 'stable'" run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" diff --git a/src/chain/cbf.rs b/src/chain/cbf.rs index d8a8d1832a..bd8f36691f 100644 --- a/src/chain/cbf.rs +++ b/src/chain/cbf.rs @@ -344,6 +344,7 @@ impl CbfChainSource { retries, e, ); + *restart_status.lock().unwrap() = CbfRuntimeStatus::Stopped; break; } log_error!( @@ -503,8 +504,8 @@ impl CbfChainSource { /// Reset filter scan state to a clean baseline. /// - /// Called on both success and error paths in `run_filter_scan()` to ensure - /// no stale state leaks between scans. + /// Called on error paths in `run_filter_scan()` to ensure no stale state + /// leaks between scans. The success path performs inline cleanup instead. fn cleanup_scan_state(&self) { self.filter_skip_height.store(0, Ordering::Release); self.watched_scripts.write().unwrap().clear(); @@ -684,11 +685,20 @@ impl CbfChainSource { // created outputs and spent inputs), so we include every transaction // from matched blocks and let BDK determine relevance. let mut tx_update = TxUpdate::default(); + let per_request_timeout = + Duration::from_secs(self.sync_config.timeouts_config.per_request_timeout_secs.into()); for (height, block_hash) in &matched { - let indexed_block = requester.get_block(*block_hash).await.map_err(|e| { - log_error!(self.logger, "Failed to fetch block {}: {:?}", block_hash, e); - Error::WalletOperationFailed - })?; + let indexed_block = + tokio::time::timeout(per_request_timeout, requester.get_block(*block_hash)) + .await + .map_err(|_| { + log_error!(self.logger, "Timed out fetching block {}", block_hash); + Error::WalletOperationFailed + })? + .map_err(|e| { + log_error!(self.logger, "Failed to fetch block {}: {:?}", block_hash, e); + Error::WalletOperationFailed + })?; let block = indexed_block.block; let block_id = BlockId { height: *height, hash: block.header.block_hash() }; let conf_time = @@ -797,12 +807,15 @@ impl CbfChainSource { // The compact block filter already matched our scripts (covering both // created outputs and spent inputs), so we confirm every transaction // from matched blocks and let LDK determine relevance. + let per_request_timeout = + Duration::from_secs(self.sync_config.timeouts_config.per_request_timeout_secs.into()); for (height, block_hash) in &matched { confirm_block_transactions( &requester, *block_hash, *height, &confirmables, + per_request_timeout, &self.logger, ) .await?; @@ -1181,12 +1194,18 @@ fn update_node_metrics_timestamp( /// Fetch a block by hash and call `transactions_confirmed` on each confirmable. async fn confirm_block_transactions( requester: &Requester, block_hash: BlockHash, height: u32, - confirmables: &[&(dyn Confirm + Sync + Send)], logger: &Logger, + confirmables: &[&(dyn Confirm + Sync + Send)], per_request_timeout: Duration, logger: &Logger, ) -> Result<(), Error> { - let indexed_block = requester.get_block(block_hash).await.map_err(|e| { - log_error!(logger, "Failed to fetch block {}: {:?}", block_hash, e); - Error::TxSyncFailed - })?; + let indexed_block = tokio::time::timeout(per_request_timeout, requester.get_block(block_hash)) + .await + .map_err(|_| { + log_error!(logger, "Timed out fetching block {}", block_hash); + Error::TxSyncFailed + })? + .map_err(|e| { + log_error!(logger, "Failed to fetch block {}: {:?}", block_hash, e); + Error::TxSyncFailed + })?; let block = &indexed_block.block; let header = &block.header; let txdata: Vec<(usize, &Transaction)> = block.txdata.iter().enumerate().collect(); diff --git a/src/config.rs b/src/config.rs index 893c734d98..2a4e12898c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -514,7 +514,7 @@ pub struct CbfSyncConfig { /// of downloading incorrect filter headers. Setting this to 1 means filter headers from a /// single peer are trusted without cross-validation. /// - /// Defaults to 2. + /// Defaults to 1. pub required_peers: u8, } @@ -524,7 +524,7 @@ impl Default for CbfSyncConfig { background_sync_config: Some(BackgroundSyncConfig::default()), timeouts_config: SyncTimeoutsConfig::default(), response_timeout_secs: 30, - required_peers: 2, + required_peers: 1, } } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4fd17499c1..a8f840bec7 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -268,7 +268,7 @@ pub(crate) fn random_chain_source<'a>( Some("bitcoind-rpc") => 2, Some("bitcoind-rest") => 3, Some("cbf") => 4, - _ => rand::random_range(0..5), + _ => rand::random_range(0..3), }; match r { 0 => {