Merge branch 'lightningdevkit:main' into lightningdevkit#2364

Author: Sharmalm

.github/workflows/build.yml

@@ -22,7 +22,6 @@ jobs:
         include:
           - toolchain: stable
             platform: ubuntu-latest
-            coverage: true
           # 1.48.0 is the MSRV for all crates except lightning-transaction-sync and Win/Mac
           - toolchain: 1.48.0
             platform: ubuntu-latest
@@ -50,46 +49,31 @@ jobs:
         run: |
           sudo apt-get -y install shellcheck
           shellcheck ci/ci-tests.sh
-      - name: Run CI script with coverage generation
-        if: matrix.coverage
-        shell: bash # Default on Winblows is powershell
-        run: LDK_COVERAGE_BUILD=true ./ci/ci-tests.sh
       - name: Run CI script
-        if: "!matrix.coverage"
         shell: bash # Default on Winblows is powershell
         run: ./ci/ci-tests.sh
-      - name: Install deps for kcov
-        if: matrix.coverage
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install binutils-dev libcurl4-openssl-dev zlib1g-dev libdw-dev libiberty-dev
-      - name: Install kcov
-        if: matrix.coverage
-        run: |
-          wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz
-          tar xzf master.tar.gz
-          cd kcov-master && mkdir build && cd build
-          cmake ..
-          make
-          make install DESTDIR=../../kcov-build
-          cd ../.. && rm -rf kcov-master master.tar.gz
-      - name: Generate coverage report
-        if: matrix.coverage
-        run: |
-          for file in target/debug/deps/lightning*; do
-            [ -x "${file}" ] || continue;
-            mkdir -p "target/cov/$(basename $file)";
-            ./kcov-build/usr/local/bin/kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$(basename $file)" "$file";
-          done
-      - name: Upload coverage
-        if: matrix.coverage
-        uses: codecov/codecov-action@v3
+
+  coverage:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v3
         with:
+          fetch-depth: 0
+      - name: Install Rust stable toolchain
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal
+      - name: Run tests with coverage generation
+        run: |
+          cargo install cargo-llvm-cov
+          export RUSTFLAGS="-Clink-dead-code -Coverflow-checks=off"
+          cargo llvm-cov --features rest-client,rpc-client,tokio,futures,serde --codecov --hide-instantiations --output-path=target/codecov.json
           # Could you use this to fake the coverage report for your PR? Sure.
           # Will anyone be impressed by your amazing coverage? No
           # Maybe if codecov wasn't broken we wouldn't need to do this...
-          token: f421b687-4dc2-4387-ac3d-dc3b2528af57
-          fail_ci_if_error: true
+          bash <(curl -s https://codecov.io/bash) -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57"
 
   benchmark:
     runs-on: ubuntu-latest

bench/benches/bench.rs

@@ -13,6 +13,9 @@ criterion_group!(benches,
 	lightning::routing::router::benches::generate_routes_with_probabilistic_scorer,
 	lightning::routing::router::benches::generate_mpp_routes_with_probabilistic_scorer,
 	lightning::routing::router::benches::generate_large_mpp_routes_with_probabilistic_scorer,
+	lightning::routing::router::benches::generate_routes_with_nonlinear_probabilistic_scorer,
+	lightning::routing::router::benches::generate_mpp_routes_with_nonlinear_probabilistic_scorer,
+	lightning::routing::router::benches::generate_large_mpp_routes_with_nonlinear_probabilistic_scorer,
 	lightning::sign::benches::bench_get_secure_random_bytes,
 	lightning::ln::channelmanager::bench::bench_sends,
 	lightning_persister::fs_store::bench::bench_sends,

ci/ci-tests.sh

@@ -29,41 +29,45 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
 # The quote crate switched to Rust edition 2021 starting with v1.0.31, i.e., has MSRV of 1.56
 [ "$RUSTC_MINOR_VERSION" -lt 56 ] && cargo update -p quote --precise "1.0.30" --verbose
 
+# The syn crate depends on too-new proc-macro2 starting with v2.0.33, i.e., has MSRV of 1.56
+if [ "$RUSTC_MINOR_VERSION" -lt 56 ]; then
+	SYN_2_DEP=$(grep -o '"syn 2.*' Cargo.lock | tr -d '",' | tr ' ' ':')
+	cargo update -p "$SYN_2_DEP" --precise "2.0.32" --verbose
+fi
+
 # The proc-macro2 crate switched to Rust edition 2021 starting with v1.0.66, i.e., has MSRV of 1.56
 [ "$RUSTC_MINOR_VERSION" -lt 56 ] && cargo update -p proc-macro2 --precise "1.0.65" --verbose
 
 # The memchr crate switched to an MSRV of 1.60 starting with v2.6.0
 [ "$RUSTC_MINOR_VERSION" -lt 60 ] && cargo update -p memchr --precise "2.5.0" --verbose
 
-[ "$LDK_COVERAGE_BUILD" != "" ] && export RUSTFLAGS="-C link-dead-code"
-
 export RUST_BACKTRACE=1
 
 echo -e "\n\nBuilding and testing all workspace crates..."
 cargo test --verbose --color always
-cargo build --verbose --color always
+cargo check --verbose --color always
 
 echo -e "\n\nBuilding and testing Block Sync Clients with features"
 pushd lightning-block-sync
 cargo test --verbose --color always --features rest-client
-cargo build --verbose --color always --features rest-client
+cargo check --verbose --color always --features rest-client
 cargo test --verbose --color always --features rpc-client
-cargo build --verbose --color always --features rpc-client
+cargo check --verbose --color always --features rpc-client
 cargo test --verbose --color always --features rpc-client,rest-client
-cargo build --verbose --color always --features rpc-client,rest-client
+cargo check --verbose --color always --features rpc-client,rest-client
 cargo test --verbose --color always --features rpc-client,rest-client,tokio
-cargo build --verbose --color always --features rpc-client,rest-client,tokio
+cargo check --verbose --color always --features rpc-client,rest-client,tokio
 popd
 
 if [[ $RUSTC_MINOR_VERSION -gt 67 && "$HOST_PLATFORM" != *windows* ]]; then
 	echo -e "\n\nBuilding and testing Transaction Sync Clients with features"
 	pushd lightning-transaction-sync
 	cargo test --verbose --color always --features esplora-blocking
-	cargo build --verbose --color always --features esplora-blocking
+	cargo check --verbose --color always --features esplora-blocking
 	cargo test --verbose --color always --features esplora-async
-	cargo build --verbose --color always --features esplora-async
+	cargo check --verbose --color always --features esplora-async
 	cargo test --verbose --color always --features esplora-async-https
-	cargo build --verbose --color always --features esplora-async-https
+	cargo check --verbose --color always --features esplora-async-https
 	popd
 fi
 
@@ -89,7 +93,7 @@ fi
 echo -e "\n\nBuilding with all Log-Limiting features"
 pushd lightning
 grep '^max_level_' Cargo.toml | awk '{ print $1 }'| while read -r FEATURE; do
-	cargo build --verbose --color always --features "$FEATURE"
+	cargo check --verbose --color always --features "$FEATURE"
 done
 popd
 
@@ -120,6 +124,10 @@ if [[ $RUSTC_MINOR_VERSION -gt 67 ]]; then
 	# lightning-transaction-sync's MSRV is 1.67
 	cargo check --verbose --color always --features lightning-transaction-sync
 else
+	# The memchr crate switched to an MSRV of 1.60 starting with v2.6.0
+	# This is currently only a release dependency via core2, which we intend to work with
+	# rust-bitcoin to remove soon.
+	[ "$RUSTC_MINOR_VERSION" -lt 60 ] && cargo update -p memchr --precise "2.5.0" --verbose
 	cargo check --verbose --color always
 fi
 popd

fuzz/src/chanmon_consistency.rs

@@ -125,7 +125,6 @@ struct TestChainMonitor {
 	// "fails" if we ever force-close a channel, we avoid doing so, always saving the latest
 	// fully-serialized monitor state here, as well as the corresponding update_id.
 	pub latest_monitors: Mutex<HashMap<OutPoint, (u64, Vec<u8>)>>,
-	pub should_update_manager: atomic::AtomicBool,
 }
 impl TestChainMonitor {
 	pub fn new(broadcaster: Arc<TestBroadcaster>, logger: Arc<dyn Logger>, feeest: Arc<FuzzEstimator>, persister: Arc<TestPersister>, keys: Arc<KeyProvider>) -> Self {
@@ -135,18 +134,16 @@ impl TestChainMonitor {
 			keys,
 			persister,
 			latest_monitors: Mutex::new(HashMap::new()),
-			should_update_manager: atomic::AtomicBool::new(false),
 		}
 	}
 }
 impl chain::Watch<TestChannelSigner> for TestChainMonitor {
-	fn watch_channel(&self, funding_txo: OutPoint, monitor: channelmonitor::ChannelMonitor<TestChannelSigner>) -> chain::ChannelMonitorUpdateStatus {
+	fn watch_channel(&self, funding_txo: OutPoint, monitor: channelmonitor::ChannelMonitor<TestChannelSigner>) -> Result<chain::ChannelMonitorUpdateStatus, ()> {
 		let mut ser = VecWriter(Vec::new());
 		monitor.write(&mut ser).unwrap();
 		if let Some(_) = self.latest_monitors.lock().unwrap().insert(funding_txo, (monitor.get_latest_update_id(), ser.0)) {
 			panic!("Already had monitor pre-watch_channel");
 		}
-		self.should_update_manager.store(true, atomic::Ordering::Relaxed);
 		self.chain_monitor.watch_channel(funding_txo, monitor)
 	}
 
@@ -162,7 +159,6 @@ impl chain::Watch<TestChannelSigner> for TestChainMonitor {
 		let mut ser = VecWriter(Vec::new());
 		deserialized_monitor.write(&mut ser).unwrap();
 		map_entry.insert((update.update_id, ser.0));
-		self.should_update_manager.store(true, atomic::Ordering::Relaxed);
 		self.chain_monitor.update_channel(funding_txo, update)
 	}
 
@@ -375,6 +371,7 @@ fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, p
 			channel_features: dest.channel_features(),
 			fee_msat: amt,
 			cltv_expiry_delta: 200,
+			maybe_announced_channel: true,
 		}], blinded_tail: None }],
 		route_params: None,
 	}, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) {
@@ -409,13 +406,15 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des
 			channel_features: middle.channel_features(),
 			fee_msat: first_hop_fee,
 			cltv_expiry_delta: 100,
+			maybe_announced_channel: true,
 		}, RouteHop {
 			pubkey: dest.get_our_node_id(),
 			node_features: dest.node_features(),
 			short_channel_id: dest_chan_id,
 			channel_features: dest.channel_features(),
 			fee_msat: amt,
 			cltv_expiry_delta: 200,
+			maybe_announced_channel: true,
 		}], blinded_tail: None }],
 		route_params: None,
 	}, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) {
@@ -501,7 +500,7 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
 			let res = (<(BlockHash, ChanMan)>::read(&mut Cursor::new(&$ser.0), read_args).expect("Failed to read manager").1, chain_monitor.clone());
 			for (funding_txo, mon) in monitors.drain() {
 				assert_eq!(chain_monitor.chain_monitor.watch_channel(funding_txo, mon),
-					ChannelMonitorUpdateStatus::Completed);
+					Ok(ChannelMonitorUpdateStatus::Completed));
 			}
 			res
 		} }
@@ -1101,11 +1100,9 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
 				if !chan_a_disconnected {
 					nodes[1].peer_disconnected(&nodes[0].get_our_node_id());
 					chan_a_disconnected = true;
-					drain_msg_events_on_disconnect!(0);
-				}
-				if monitor_a.should_update_manager.load(atomic::Ordering::Relaxed) {
-					node_a_ser.0.clear();
-					nodes[0].write(&mut node_a_ser).unwrap();
+					push_excess_b_events!(nodes[1].get_and_clear_pending_msg_events().drain(..), Some(0));
+					ab_events.clear();
+					ba_events.clear();
 				}
 				let (new_node_a, new_monitor_a) = reload_node!(node_a_ser, 0, monitor_a, keys_manager_a, fee_est_a);
 				nodes[0] = new_node_a;
@@ -1134,11 +1131,9 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
 				if !chan_b_disconnected {
 					nodes[1].peer_disconnected(&nodes[2].get_our_node_id());
 					chan_b_disconnected = true;
-					drain_msg_events_on_disconnect!(2);
-				}
-				if monitor_c.should_update_manager.load(atomic::Ordering::Relaxed) {
-					node_c_ser.0.clear();
-					nodes[2].write(&mut node_c_ser).unwrap();
+					push_excess_b_events!(nodes[1].get_and_clear_pending_msg_events().drain(..), Some(2));
+					bc_events.clear();
+					cb_events.clear();
 				}
 				let (new_node_c, new_monitor_c) = reload_node!(node_c_ser, 2, monitor_c, keys_manager_c, fee_est_c);
 				nodes[2] = new_node_c;
@@ -1304,15 +1299,18 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
 			_ => test_return!(),
 		}
 
-		node_a_ser.0.clear();
-		nodes[0].write(&mut node_a_ser).unwrap();
-		monitor_a.should_update_manager.store(false, atomic::Ordering::Relaxed);
-		node_b_ser.0.clear();
-		nodes[1].write(&mut node_b_ser).unwrap();
-		monitor_b.should_update_manager.store(false, atomic::Ordering::Relaxed);
-		node_c_ser.0.clear();
-		nodes[2].write(&mut node_c_ser).unwrap();
-		monitor_c.should_update_manager.store(false, atomic::Ordering::Relaxed);
+		if nodes[0].get_and_clear_needs_persistence() == true {
+			node_a_ser.0.clear();
+			nodes[0].write(&mut node_a_ser).unwrap();
+		}
+		if nodes[1].get_and_clear_needs_persistence() == true {
+			node_b_ser.0.clear();
+			nodes[1].write(&mut node_b_ser).unwrap();
+		}
+		if nodes[2].get_and_clear_needs_persistence() == true {
+			node_c_ser.0.clear();
+			nodes[2].write(&mut node_c_ser).unwrap();
+		}
 	}
 }
 

fuzz/src/onion_hop_data.rs

@@ -11,18 +11,21 @@
 // To modify it, modify msg_target_template.txt and run gen_target.sh instead.
 
 use crate::utils::test_logger;
+use lightning::util::test_utils;
 
 #[inline]
 pub fn onion_hop_data_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	use lightning::util::ser::Readable;
+	use lightning::util::ser::ReadableArgs;
 	let mut r = ::std::io::Cursor::new(data);
-	let _ =  <lightning::ln::msgs::InboundOnionPayload as Readable>::read(&mut r);
+	let node_signer = test_utils::TestNodeSigner::new(test_utils::privkey(42));
+	let _ =  <lightning::ln::msgs::InboundOnionPayload as ReadableArgs<&&test_utils::TestNodeSigner>>::read(&mut r, &&node_signer);
 }
 
 #[no_mangle]
 pub extern "C" fn onion_hop_data_run(data: *const u8, datalen: usize) {
-	use lightning::util::ser::Readable;
+	use lightning::util::ser::ReadableArgs;
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
 	let mut r = ::std::io::Cursor::new(data);
-	let _ =  <lightning::ln::msgs::InboundOnionPayload as Readable>::read(&mut r);
+	let node_signer = test_utils::TestNodeSigner::new(test_utils::privkey(42));
+	let _ =  <lightning::ln::msgs::InboundOnionPayload as ReadableArgs<&&test_utils::TestNodeSigner>>::read(&mut r, &&node_signer);
 }

lightning-background-processor/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-background-processor"
-version = "0.0.116"
+version = "0.0.117-alpha1"
 authors = ["Valentine Wallace <[email protected]>"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning"
@@ -22,11 +22,11 @@ default = ["std"]
 
 [dependencies]
 bitcoin = { version = "0.29.0", default-features = false }
-lightning = { version = "0.0.116", path = "../lightning", default-features = false }
-lightning-rapid-gossip-sync = { version = "0.0.116", path = "../lightning-rapid-gossip-sync", default-features = false }
+lightning = { version = "0.0.117-alpha1", path = "../lightning", default-features = false }
+lightning-rapid-gossip-sync = { version = "0.0.117-alpha1", path = "../lightning-rapid-gossip-sync", default-features = false }
 
 [dev-dependencies]
 tokio = { version = "1.14", features = [ "macros", "rt", "rt-multi-thread", "sync", "time" ] }
-lightning = { version = "0.0.116", path = "../lightning", features = ["_test_utils"] }
-lightning-invoice = { version = "0.24.0", path = "../lightning-invoice" }
-lightning-persister = { version = "0.0.116", path = "../lightning-persister" }
+lightning = { version = "0.0.117-alpha1", path = "../lightning", features = ["_test_utils"] }
+lightning-invoice = { version = "0.25.0-alpha1", path = "../lightning-invoice" }
+lightning-persister = { version = "0.0.117-alpha1", path = "../lightning-persister" }