Merge branch 'master' into modular_handshake

Author: arik-so

fuzz/README.md

@@ -0,0 +1,86 @@
+# Fuzzing
+
+Fuzz tests generate a ton of random parameter arguments to the program and then validate that none cause it to crash.
+
+## How does it work?
+
+Typically, Travis CI will run `travis-fuzz.sh` on one of the environments the automated tests are configured for.
+This is the most time-consuming component of the continuous integration workflow, so it is recommended that you detect
+issues locally, and Travis merely acts as a sanity check. Fuzzing is further only effective with
+a lot of CPU time, indicating that if crash scenarios are discovered on Travis with its low
+runtime constraints, the crash is caused relatively easily.
+
+## How do I run fuzz tests locally?
+
+You typically won't need to run the entire combination of different fuzzing tools. For local execution, `honggfuzz`
+should be more than sufficient. 
+
+### Setup
+
+To install `honggfuzz`, simply run
+
+```shell
+cargo update
+cargo install --force honggfuzz
+```
+
+### Execution
+
+To run the Hongg fuzzer, do
+
+```shell
+export CPU_COUNT=1 # replace as needed
+export HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz"
+export HFUZZ_RUN_ARGS="-n $CPU_COUNT --exit_upon_crash"
+
+export TARGET="msg_ping_target" # replace with the target to be fuzzed
+cargo hfuzz run $TARGET 
+```
+
+To see a list of available fuzzing targets, run:
+
+```shell
+ls ./src/bin/
+```
+
+## A fuzz test failed on Travis, what do I do?
+
+You're trying to create a PR, but need to find the underlying cause of that pesky fuzz failure blocking the merge?
+
+Worry not, for this is easily traced.
+
+If your Travis output log looks like this:
+
+```
+Size:639 (i,b,hw,ed,ip,cmp): 0/0/0/0/0/1, Tot:0/0/0/2036/5/28604
+Seen a crash. Terminating all fuzzing threads
+
+… # a lot of lines in between
+
+<0x0000555555565559> [func:UNKNOWN file: line:0 module:/home/travis/build/rust-bitcoin/rust-lightning/fuzz/hfuzz_target/x86_64-unknown-linux-gnu/release/full_stack_target]
+<0x0000000000000000> [func:UNKNOWN file: line:0 module:UNKNOWN]
+=====================================================================
+2d3136383734090101010101010101010101010101010101010101010101
+010101010100040101010101010101010101010103010101010100010101
+0069d07c319a4961
+The command "if [ "$(rustup show | grep default | grep stable)" != "" ]; then cd fuzz && cargo test --verbose && ./travis-fuzz.sh; fi" exited with 1.
+```
+
+Note that the penultimate stack trace line ends in `release/full_stack_target]`. That indicates that
+the failing target was `full_stack`. To reproduce the error locally, simply copy the hex, 
+and run the following from the `fuzz` directory:
+
+```shell
+export TARGET="full_stack" # adjust for your output
+export HEX="2d3136383734090101010101010101010101010101010101010101010101\
+010101010100040101010101010101010101010103010101010100010101\
+0069d07c319a4961" # adjust for your output
+
+mkdir -p ./test_cases/$TARGET
+echo $HEX | xxd -r -p > ./test_cases/$TARGET/any_filename_works
+
+export RUST_BACKTRACE=1
+cargo test
+```
+
+This will reproduce the failing fuzz input and yield a usable stack trace.

lightning-net-tokio/src/lib.rs

@@ -507,7 +507,7 @@ mod tests {
 		fn handle_channel_announcement(&self, _msg: &ChannelAnnouncement) -> Result<bool, LightningError> { Ok(false) }
 		fn handle_channel_update(&self, _msg: &ChannelUpdate) -> Result<bool, LightningError> { Ok(false) }
 		fn handle_htlc_fail_channel_update(&self, _update: &HTLCFailChannelUpdate) { }
-		fn get_next_channel_announcements(&self, _starting_point: u64, _batch_amount: u8) -> Vec<(ChannelAnnouncement, ChannelUpdate, ChannelUpdate)> { Vec::new() }
+		fn get_next_channel_announcements(&self, _starting_point: u64, _batch_amount: u8) -> Vec<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)> { Vec::new() }
 		fn get_next_node_announcements(&self, _starting_point: Option<&PublicKey>, _batch_amount: u8) -> Vec<NodeAnnouncement> { Vec::new() }
 		fn should_request_full_sync(&self, _node_id: &PublicKey) -> bool { false }
 	}

lightning/src/chain/chaininterface.rs

@@ -22,6 +22,7 @@ use std::marker::PhantomData;
 use std::ptr;
 
 /// Used to give chain error details upstream
+#[derive(Clone)]
 pub enum ChainError {
 	/// Client doesn't support UTXO lookup (but the chain hash matches our genesis block hash)
 	NotSupported,

lightning/src/ln/msgs.rs

@@ -601,7 +601,7 @@ pub trait RoutingMessageHandler : Send + Sync {
 	/// Gets a subset of the channel announcements and updates required to dump our routing table
 	/// to a remote node, starting at the short_channel_id indicated by starting_point and
 	/// including the batch_amount entries immediately higher in numerical value than starting_point.
-	fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, ChannelUpdate, ChannelUpdate)>;
+	fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)>;
 	/// Gets a subset of the node announcements required to dump our routing table to a remote node,
 	/// starting at the node *after* the provided publickey and including batch_amount entries
 	/// immediately higher (as defined by <PublicKey as Ord>::cmp) than starting_point.

lightning/src/ln/peer_handler.rs

@@ -367,10 +367,14 @@ impl<Descriptor: SocketDescriptor, CM: Deref> PeerManager<Descriptor, CM> where
 					InitSyncTracker::ChannelsSyncing(c) if c < 0xffff_ffff_ffff_ffff => {
 						let steps = ((MSG_BUFF_SIZE - peer.pending_outbound_buffer.len() + 2) / 3) as u8;
 						let all_messages = self.message_handler.route_handler.get_next_channel_announcements(c, steps);
-						for &(ref announce, ref update_a, ref update_b) in all_messages.iter() {
+						for &(ref announce, ref update_a_option, ref update_b_option) in all_messages.iter() {
 							encode_and_send_msg!(announce);
-							encode_and_send_msg!(update_a);
-							encode_and_send_msg!(update_b);
+							if let &Some(ref update_a) = update_a_option {
+								encode_and_send_msg!(update_a);
+							}
+							if let &Some(ref update_b) = update_b_option {
+								encode_and_send_msg!(update_b);
+							}
 							peer.sync_status = InitSyncTracker::ChannelsSyncing(announce.contents.short_channel_id + 1);
 						}
 						if all_messages.is_empty() || all_messages.len() != steps as usize {
@@ -1364,7 +1368,7 @@ mod tests {
 			Err(msgs::LightningError { err: "", action: msgs::ErrorAction::IgnoreError })
 		}
 		fn handle_htlc_fail_channel_update(&self, _update: &msgs::HTLCFailChannelUpdate) {}
-		fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(msgs::ChannelAnnouncement, msgs::ChannelUpdate,msgs::ChannelUpdate)> {
+		fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(msgs::ChannelAnnouncement, Option<msgs::ChannelUpdate>, Option<msgs::ChannelUpdate>)> {
 			let mut chan_anns = Vec::new();
 			const TOTAL_UPDS: u64 = 100;
 			let end: u64 =  min(starting_point + batch_amount as u64, TOTAL_UPDS - self.chan_anns_sent.load(Ordering::Acquire) as u64);
@@ -1373,7 +1377,7 @@ mod tests {
 				let chan_upd_2 = get_dummy_channel_update(i);
 				let chan_ann = get_dummy_channel_announcement(i);
 
-				chan_anns.push((chan_ann, chan_upd_1, chan_upd_2));
+				chan_anns.push((chan_ann, Some(chan_upd_1), Some(chan_upd_2)));
 			}
 
 			self.chan_anns_sent.fetch_add(chan_anns.len(), Ordering::AcqRel);