Merge remote-tracking branch 'upstream/master' into modular_handshake

# Conflicts:
#	fuzz/src/full_stack.rs
#	lightning/src/ln/peers/handler.rs

Author: arik-so

.editorconfig

@@ -2,3 +2,4 @@
 
 [*]
 indent_style = tab
+insert_final_newline = true

.github/workflows/build.yml

@@ -0,0 +1,96 @@
+name: Continuous Integration Checks
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        toolchain: [ stable,
+                     beta,
+                     # 1.22.0 is MSRV for rust-lightning in general:
+                     1.22.0,
+                     # 1.34.2 is Debian stable
+                     1.34.2,
+                     # 1.39.0 is MSRV for lightning-net-tokio and generates coverage
+                     1.39.0]
+        include:
+          - toolchain: stable
+            build-net-tokio: true
+          - toolchain: beta
+            build-net-tokio: true
+          - toolchain: 1.39.0
+            build-net-tokio: true
+            coverage: true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+      - name: Install Rust ${{ matrix.toolchain }} toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ matrix.toolchain }}
+          override: true
+          profile: minimal
+      - name: Build on Rust ${{ matrix.toolchain }} with net-tokio
+        if: matrix.build-net-tokio
+        run: RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always
+      - name: Build on Rust ${{ matrix.toolchain }}
+        if: "! matrix.build-net-tokio"
+        run: RUSTFLAGS="-C link-dead-code" cargo build --verbose  --color always -p lightning
+      - name: Test on Rust ${{ matrix.toolchain }} with net-tokio
+        if: matrix.build-net-tokio
+        run: RUSTFLAGS="-C link-dead-code" cargo test --verbose --color always
+      - name: Test on Rust ${{ matrix.toolchain }}
+        if: "! matrix.build-net-tokio"
+        run: RUSTFLAGS="-C link-dead-code" cargo test --verbose --color always  -p lightning
+      - name: Install deps for kcov
+        if: matrix.coverage
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install binutils-dev libcurl4-openssl-dev zlib1g-dev libdw-dev libiberty-dev
+      - name: Install kcov
+        if: matrix.coverage
+        run: |
+          wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz
+          tar xzf master.tar.gz
+          cd kcov-master && mkdir build && cd build
+          cmake ..
+          make
+          make install DESTDIR=../../kcov-build
+          cd ../.. && rm -rf kcov-master master.tar.gz
+      - name: Generate coverage report
+        if: matrix.coverage
+        run: |
+          for file in target/debug/lightning-*; do
+            [ -x "${file}" ] || continue;
+            mkdir -p "target/cov/$(basename $file)";
+            ./kcov-build/usr/local/bin/kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$(basename $file)" "$file";
+          done
+      - name: Upload coverage
+        if: matrix.coverage
+        uses: codecov/codecov-action@v1
+        with:
+          fail_ci_if_error: true
+
+  fuzz:
+    runs-on: ubuntu-latest
+    env:
+      TOOLCHAIN: stable
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+      - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.TOOLCHAIN }}
+          override: true
+          profile: minimal
+      - name: Install dependencies for honggfuzz
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install build-essential binutils-dev libunwind-dev
+      - name: Sanity check fuzz targets on Rust ${{ env.TOOLCHAIN }}
+        run: cd fuzz && cargo test --verbose --color always
+      - name: Run fuzzers
+        run: cd fuzz && ./ci-fuzz.sh

.travis.yml

@@ -26,7 +26,7 @@ script:
      - if [ "$BUILD_NET_TOKIO" == "1" ]; then RUSTFLAGS="-C link-dead-code" cargo test --verbose; fi
      - if [ "$BUILD_NET_TOKIO" != "1" ]; then RUSTFLAGS="-C link-dead-code" cargo test --verbose -p lightning; fi
      # Run lightning workspace fuzz tests on Rust stable
-     - if [ "$(rustup show | grep default | grep stable)" != "" ]; then cd fuzz && cargo test --verbose && ./travis-fuzz.sh; fi
+     - if [ "$(rustup show | grep default | grep stable)" != "" ]; then cd fuzz && cargo test --verbose && ./ci-fuzz.sh; fi
      # Generate code cov information on Rust 1.39.0
      - if [ "$(rustup show | grep default | grep 1.39.0)" != "" ]; then
            wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz &&

ARCH.md

@@ -11,8 +11,8 @@ receive `ChannelMonitorUpdate`s from `ChannelManager` and persist them to disk b
 channel steps forward.
 
 There are two additional important structures that you may use either on the same device
-as the `ChannelManager` or on a separate one. `Router` handles receiving channel and node
-node announcements and calculates routes for sending payments. `PeerManager` handles the
+as the `ChannelManager` or on a separate one. `Router` handles receiving channel and node 
+announcements and calculates routes for sending payments. `PeerManager` handles the
 authenticated and encrypted communication protocol, monitoring for liveness of peers,
 routing messages to `ChannelManager` and `Router` instances directly, and receiving
 messages from them via the `EventsProvider` interface.

fuzz/Cargo.toml

@@ -19,11 +19,9 @@ stdin_fuzz = []
 [dependencies]
 afl = { version = "0.4", optional = true }
 lightning = { path = "../lightning", features = ["fuzztarget"] }
-bitcoin = { version = "0.21", features = ["fuzztarget"] }
-bitcoin_hashes = { version = "0.7", features = ["fuzztarget"] }
+bitcoin = { version = "0.23", features = ["fuzztarget"] }
 hex = "0.3"
 honggfuzz = { version = "0.5", optional = true }
-secp256k1 = { version = "0.15", features=["fuzztarget"] }
 libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git", optional = true }
 
 [build-dependencies]
@@ -37,6 +35,10 @@ members = ["."]
 lto = true
 codegen-units = 1
 
+# When testing a large fuzz corpus, -O1 offers a nice speedup
+[profile.dev]
+opt-level = 1
+
 [lib]
 name = "lightning_fuzz"
 path = "src/lib.rs"

fuzz/README.md

@@ -83,4 +83,4 @@ export RUST_BACKTRACE=1
 cargo test
 ```
 
-This will reproduce the failing fuzz input and yield a usable stack trace.
+This will reproduce the failing fuzz input and yield a usable stack trace.

fuzz/travis-fuzz.sh renamed to fuzz/ci-fuzz.sh

@@ -12,20 +12,22 @@ rm *_target.rs
 [ "$(git diff)" != "" ] && exit 1
 popd
 
-cargo install --force honggfuzz
+cargo install --color always --force honggfuzz
 sed -i 's/lto = true//' Cargo.toml
-HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" cargo hfuzz build
+HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" cargo --color always hfuzz build
 for TARGET in src/bin/*.rs; do
 	FILENAME=$(basename $TARGET)
 	FILE="${FILENAME%.*}"
 	HFUZZ_RUN_ARGS="--exit_upon_crash -v -n2"
 	if [ "$FILE" = "chanmon_consistency_target" ]; then
 		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -F 64 -N100000"
+	elif [ "$FILE" = "full_stack_target" ]; then
+		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -t0 -N1000000"
 	else
 		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -N1000000"
 	fi
 	export HFUZZ_RUN_ARGS
-	HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" cargo hfuzz run $FILE
+	HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" cargo --color always hfuzz run $FILE
 	if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then
 		cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT
 		for CASE in hfuzz_workspace/$FILE/SIG*; do

fuzz/src/bin/chanmon_consistency_target.rs

@@ -6,9 +6,6 @@
 extern crate lightning_fuzz;
 use lightning_fuzz::chanmon_consistency::*;
 
-use std::fs;
-use std::io::Read;
-
 #[cfg(feature = "afl")]
 #[macro_use] extern crate afl;
 #[cfg(feature = "afl")]
@@ -38,22 +35,59 @@ fuzz_target!(|data: &[u8]| {
 
 #[cfg(feature = "stdin_fuzz")]
 fn main() {
+	use std::io::Read;
+
 	let mut data = Vec::with_capacity(8192);
 	std::io::stdin().read_to_end(&mut data).unwrap();
 	chanmon_consistency_run(data.as_ptr(), data.len());
 }
 
 #[test]
 fn run_test_cases() {
-	let mut data: Vec<u8> = vec![0];
-	chanmon_consistency_run(data.as_ptr(), data.len());
+	use std::fs;
+	use std::io::Read;
+	use lightning_fuzz::utils::test_logger::StringBuffer;
+
+	use std::sync::{atomic, Arc};
+	{
+		let data: Vec<u8> = vec![0];
+		chanmon_consistency_run(data.as_ptr(), data.len());
+	}
+	let mut threads = Vec::new();
+	let threads_running = Arc::new(atomic::AtomicUsize::new(0));
 	if let Ok(tests) = fs::read_dir("test_cases/chanmon_consistency") {
 		for test in tests {
-			data.clear();
+			let mut data: Vec<u8> = Vec::new();
 			let path = test.unwrap().path();
-			println!("Running test {}...", path.file_name().unwrap().to_str().unwrap());
-			fs::File::open(path).unwrap().read_to_end(&mut data).unwrap();
-			chanmon_consistency_run(data.as_ptr(), data.len());
+			fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
+			threads_running.fetch_add(1, atomic::Ordering::AcqRel);
+
+			let thread_count_ref = Arc::clone(&threads_running);
+			let main_thread_ref = std::thread::current();
+			threads.push((path.file_name().unwrap().to_str().unwrap().to_string(),
+				std::thread::spawn(move || {
+					let string_logger = StringBuffer::new();
+
+					let panic_logger = string_logger.clone();
+					let res = if ::std::panic::catch_unwind(move || {
+						chanmon_consistency_test(&data, panic_logger);
+					}).is_err() {
+						Some(string_logger.into_string())
+					} else { None };
+					thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel);
+					main_thread_ref.unpark();
+					res
+				})
+			));
+			while threads_running.load(atomic::Ordering::Acquire) > 32 {
+				std::thread::park();
+			}
+		}
+	}
+	for (test, thread) in threads.drain(..) {
+		if let Some(output) = thread.join().unwrap() {
+			println!("Output of {}:\n{}", test, output);
+			panic!();
 		}
 	}
 }

fuzz/src/bin/chanmon_deser_target.rs

@@ -6,9 +6,6 @@
 extern crate lightning_fuzz;
 use lightning_fuzz::chanmon_deser::*;
 
-use std::fs;
-use std::io::Read;
-
 #[cfg(feature = "afl")]
 #[macro_use] extern crate afl;
 #[cfg(feature = "afl")]
@@ -38,22 +35,59 @@ fuzz_target!(|data: &[u8]| {
 
 #[cfg(feature = "stdin_fuzz")]
 fn main() {
+	use std::io::Read;
+
 	let mut data = Vec::with_capacity(8192);
 	std::io::stdin().read_to_end(&mut data).unwrap();
 	chanmon_deser_run(data.as_ptr(), data.len());
 }
 
 #[test]
 fn run_test_cases() {
-	let mut data: Vec<u8> = vec![0];
-	chanmon_deser_run(data.as_ptr(), data.len());
+	use std::fs;
+	use std::io::Read;
+	use lightning_fuzz::utils::test_logger::StringBuffer;
+
+	use std::sync::{atomic, Arc};
+	{
+		let data: Vec<u8> = vec![0];
+		chanmon_deser_run(data.as_ptr(), data.len());
+	}
+	let mut threads = Vec::new();
+	let threads_running = Arc::new(atomic::AtomicUsize::new(0));
 	if let Ok(tests) = fs::read_dir("test_cases/chanmon_deser") {
 		for test in tests {
-			data.clear();
+			let mut data: Vec<u8> = Vec::new();
 			let path = test.unwrap().path();
-			println!("Running test {}...", path.file_name().unwrap().to_str().unwrap());
-			fs::File::open(path).unwrap().read_to_end(&mut data).unwrap();
-			chanmon_deser_run(data.as_ptr(), data.len());
+			fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
+			threads_running.fetch_add(1, atomic::Ordering::AcqRel);
+
+			let thread_count_ref = Arc::clone(&threads_running);
+			let main_thread_ref = std::thread::current();
+			threads.push((path.file_name().unwrap().to_str().unwrap().to_string(),
+				std::thread::spawn(move || {
+					let string_logger = StringBuffer::new();
+
+					let panic_logger = string_logger.clone();
+					let res = if ::std::panic::catch_unwind(move || {
+						chanmon_deser_test(&data, panic_logger);
+					}).is_err() {
+						Some(string_logger.into_string())
+					} else { None };
+					thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel);
+					main_thread_ref.unpark();
+					res
+				})
+			));
+			while threads_running.load(atomic::Ordering::Acquire) > 32 {
+				std::thread::park();
+			}
+		}
+	}
+	for (test, thread) in threads.drain(..) {
+		if let Some(output) = thread.join().unwrap() {
+			println!("Output of {}:\n{}", test, output);
+			panic!();
 		}
 	}
 }