Merge pull request #220 from tnull/2023-12-homogenous-ci

Make CI scripts more homogeneous

Author: tnull

.github/workflows/cln.yml renamed to .github/workflows/cln-integration.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration Checks - CLN
+name: CI Checks - CLN Integration Tests
 
 on: [push, pull_request]
 

.github/workflows/kotlin.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration Checks - Kotlin
+name: CI Checks - Kotlin Tests
 
 on: [push, pull_request]
 

.github/workflows/python.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration Checks - Python
+name: CI Checks - Python Tests
 
 on: [push, pull_request]
 

.github/workflows/build.yml renamed to .github/workflows/rust.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration Checks
+name: CI Checks - Rust Tests
 
 on: [push, pull_request]
 
@@ -46,13 +46,18 @@ jobs:
           cargo update -p proptest --precise "1.2.0" --verbose # proptest 1.3.0 requires rustc 1.64.0
           cargo update -p reqwest --precise "0.11.20" --verbose # reqwest 0.11.21 broke 1.63.0 MSRV
           cargo update -p regex --precise "1.9.6" --verbose # regex 1.10.0 requires rustc 1.65.0
-          cargo update -p jobserver --precise "0.1.26" --verbose # jobserver 0.1.27 requires rustc 1.66.0
-          cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5" --verbose # zstd-sys 2.0.9+zstd.1.5.5 requires rustc 1.64.0
-          cargo update -p petgraph --precise "0.6.3" --verbose # petgraph v0.6.4, requires rustc 1.64 or newer
           cargo update -p home --precise "0.5.5" --verbose # home v0.5.9, requires rustc 1.70 or newer
       - name: Set RUSTFLAGS to deny warnings
         if: "matrix.toolchain == 'stable'"
         run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
+      - name: Download bitcoind/electrs and set environment variables
+        if: "matrix.platform != 'windows-latest'"
+        run: |
+          source ./scripts/download_bitcoind_electrs.sh
+          cp "$BITCOIND_EXE" "$HOME"/bitcoind
+          cp "$ELECTRS_EXE" "$HOME"/electrs
+          echo "BITCOIND_EXE=$HOME/bitcoind" >> "$GITHUB_ENV"
+          echo "ELECTRS_EXE=$HOME/electrs" >> "$GITHUB_ENV"
       - name: Build on Rust ${{ matrix.toolchain }}
         run: cargo build --verbose --color always
       - name: Build with UniFFI support on Rust ${{ matrix.toolchain }}
@@ -69,10 +74,12 @@ jobs:
         run: cargo check --release --features uniffi --verbose --color always
       - name: Test on Rust ${{ matrix.toolchain }}
         if: "matrix.platform != 'windows-latest'"
-        run: cargo test
+        run: |
+          RUSTFLAGS="--cfg no_download" cargo test
       - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
         if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
-        run: cargo test --features uniffi
+        run: |
+          RUSTFLAGS="--cfg no_download" cargo test --features uniffi
       - name: Check formatting on Rust ${{ matrix.toolchain }}
         if: matrix.check-fmt
         run: rustup component add rustfmt && cargo fmt --all -- --check

.github/workflows/vss-integration.yml

@@ -1,4 +1,4 @@
-name: VSS Integration Test
+name: CI Checks - VSS Integration Tests
 
 on: [push, pull_request]
 
@@ -74,8 +74,8 @@ jobs:
         run: |
           cd ldk-node
           export TEST_VSS_BASE_URL="http://localhost:8080/vss"
-          RUSTFLAGS="--cfg=vss_test --cfg=vss" cargo build --verbose --color always
-          RUSTFLAGS="--cfg=vss_test --cfg=vss" cargo test -- --nocapture
+          RUSTFLAGS="--cfg vss_test --cfg vss" cargo build --verbose --color always
+          RUSTFLAGS="--cfg vss_test --cfg vss" cargo test --test integration_tests_vss
 
       - name: Cleanup
         run: |

Cargo.toml

@@ -76,14 +76,19 @@ winapi = { version = "0.3", features = ["winbase"] }
 [dev-dependencies]
 lightning = { version = "0.0.119", features = ["std", "_test_utils"] }
 #lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch="main", features = ["std", "_test_utils"] }
-electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
-electrum-client = { version = "0.18.0", default-features = true }
+electrum-client = { version = "0.15.1", default-features = true }
+bitcoincore-rpc = { version = "0.17.0", default-features = false }
 proptest = "1.0.0"
 regex = "1.5.6"
 
+[target.'cfg(not(no_download))'.dev-dependencies]
+electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
+
+[target.'cfg(no_download)'.dev-dependencies]
+electrsd = { version = "0.26.0", features = ["legacy"] }
+
 [target.'cfg(cln_test)'.dev-dependencies]
 clightningrpc = { version = "0.3.0-beta.8", default-features = false }
-bitcoincore-rpc = { version = "0.17.0", default-features = false }
 
 [build-dependencies]
 uniffi = { version = "0.25.1", features = ["build"], optional = true }

rustfmt.toml

@@ -2,4 +2,4 @@ hard_tabs = true # use tab characters for indentation, spaces for alignment
 use_field_init_shorthand = true
 max_width = 100
 use_small_heuristics = "Max"
-fn_args_layout = "Compressed"
+fn_params_layout = "Compressed"

scripts/download_bitcoind_electrs.sh

@@ -0,0 +1,43 @@
+# Our Esplora-based tests require `electrs` and `bitcoind`
+# binaries. Here, we download the binaries, validate them, and export their
+# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
+# `electrsd`/`bitcoind` crates in our tests.
+
+HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')"
+ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
+ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
+BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
+BITCOIND_VERSION="25.1"
+if [[ "$HOST_PLATFORM" == *linux* ]]; then
+	ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
+	ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
+	BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
+	BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
+elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
+	ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
+	ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
+	BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
+	BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
+else
+	echo "\n\nUnsupported platform: $HOST_PLATFORM Exiting.."
+	exit 1
+fi
+
+DL_TMP_DIR=$(mktemp -d)
+trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
+
+pushd "$DL_TMP_DIR"
+ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
+curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
+echo "$ELECTRS_DL_HASH  $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
+unzip "$ELECTRS_DL_FILE_NAME"
+export ELECTRS_EXE="$DL_TMP_DIR"/electrs
+chmod +x "$ELECTRS_EXE"
+
+BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
+curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
+echo "$BITCOIND_DL_HASH  $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
+tar xzf "$BITCOIND_DL_FILE_NAME"
+export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
+chmod +x "$BITCOIND_EXE"
+popd

src/event.rs

@@ -795,8 +795,7 @@ where
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::test::utils::TestLogger;
-	use lightning::util::test_utils::TestStore;
+	use lightning::util::test_utils::{TestLogger, TestStore};
 
 	#[test]
 	fn event_queue_persistence() {

src/io/sqlite_store/migrations.rs

@@ -69,8 +69,7 @@ pub(super) fn migrate_schema(
 #[cfg(test)]
 mod tests {
 	use crate::io::sqlite_store::SqliteStore;
-	use crate::io::test_utils::do_read_write_remove_list_persist;
-	use crate::test::utils::random_storage_path;
+	use crate::io::test_utils::{do_read_write_remove_list_persist, random_storage_path};
 
 	use lightning::util::persist::KVStore;
 

src/io/sqlite_store/mod.rs

@@ -284,8 +284,9 @@ impl KVStore for SqliteStore {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::io::test_utils::{do_read_write_remove_list_persist, do_test_store};
-	use crate::test::utils::random_storage_path;
+	use crate::io::test_utils::{
+		do_read_write_remove_list_persist, do_test_store, random_storage_path,
+	};
 
 	impl Drop for SqliteStore {
 		fn drop(&mut self) {

src/io/test_utils.rs

@@ -1,6 +1,3 @@
-use crate::io::sqlite_store::SqliteStore;
-use lightning_persister::fs_store::FilesystemStore;
-
 use lightning::ln::functional_test_utils::{
 	connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block,
 	create_network, create_node_cfgs, create_node_chanmgrs, send_payment,
@@ -9,12 +6,22 @@ use lightning::util::persist::{read_channel_monitors, KVStore, KVSTORE_NAMESPACE
 
 use lightning::chain::channelmonitor::CLOSED_CHANNEL_UPDATE_ID;
 use lightning::events::ClosureReason;
-use lightning::util::test_utils::{self, TestStore};
+use lightning::util::test_utils;
 use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event};
 
+use rand::distributions::Alphanumeric;
+use rand::{thread_rng, Rng};
+
 use std::panic::RefUnwindSafe;
 use std::path::PathBuf;
-use std::sync::RwLock;
+
+pub(crate) fn random_storage_path() -> PathBuf {
+	let mut temp_path = std::env::temp_dir();
+	let mut rng = thread_rng();
+	let rand_dir: String = (0..7).map(|_| rng.sample(Alphanumeric) as char).collect();
+	temp_path.push(rand_dir);
+	temp_path
+}
 
 pub(crate) fn do_read_write_remove_list_persist<K: KVStore + RefUnwindSafe>(kv_store: &K) {
 	let data = [42u8; 32];
@@ -173,148 +180,3 @@ pub(crate) fn do_test_store<K: KVStore>(store_0: &K, store_1: &K) {
 	// Make sure everything is persisted as expected after close.
 	check_persisted_data!(CLOSED_CHANNEL_UPDATE_ID);
 }
-
-// A `KVStore` impl for testing purposes that wraps all our `KVStore`s and asserts their synchronicity.
-pub(crate) struct TestSyncStore {
-	serializer: RwLock<()>,
-	test_store: TestStore,
-	fs_store: FilesystemStore,
-	sqlite_store: SqliteStore,
-}
-
-impl TestSyncStore {
-	pub(crate) fn new(dest_dir: PathBuf) -> Self {
-		let serializer = RwLock::new(());
-		let mut fs_dir = dest_dir.clone();
-		fs_dir.push("fs_store");
-		let fs_store = FilesystemStore::new(fs_dir);
-		let mut sql_dir = dest_dir.clone();
-		sql_dir.push("sqlite_store");
-		let sqlite_store = SqliteStore::new(
-			sql_dir,
-			Some("test_sync_db".to_string()),
-			Some("test_sync_table".to_string()),
-		)
-		.unwrap();
-		let test_store = TestStore::new(false);
-		Self { serializer, fs_store, sqlite_store, test_store }
-	}
-
-	fn do_list(
-		&self, primary_namespace: &str, secondary_namespace: &str,
-	) -> std::io::Result<Vec<String>> {
-		let fs_res = self.fs_store.list(primary_namespace, secondary_namespace);
-		let sqlite_res = self.sqlite_store.list(primary_namespace, secondary_namespace);
-		let test_res = self.test_store.list(primary_namespace, secondary_namespace);
-
-		match fs_res {
-			Ok(mut list) => {
-				list.sort();
-
-				let mut sqlite_list = sqlite_res.unwrap();
-				sqlite_list.sort();
-				assert_eq!(list, sqlite_list);
-
-				let mut test_list = test_res.unwrap();
-				test_list.sort();
-				assert_eq!(list, test_list);
-
-				Ok(list)
-			}
-			Err(e) => {
-				assert!(sqlite_res.is_err());
-				assert!(test_res.is_err());
-				Err(e)
-			}
-		}
-	}
-}
-
-impl KVStore for TestSyncStore {
-	fn read(
-		&self, primary_namespace: &str, secondary_namespace: &str, key: &str,
-	) -> std::io::Result<Vec<u8>> {
-		let _guard = self.serializer.read().unwrap();
-
-		let fs_res = self.fs_store.read(primary_namespace, secondary_namespace, key);
-		let sqlite_res = self.sqlite_store.read(primary_namespace, secondary_namespace, key);
-		let test_res = self.test_store.read(primary_namespace, secondary_namespace, key);
-
-		match fs_res {
-			Ok(read) => {
-				assert_eq!(read, sqlite_res.unwrap());
-				assert_eq!(read, test_res.unwrap());
-				Ok(read)
-			}
-			Err(e) => {
-				assert!(sqlite_res.is_err());
-				assert_eq!(e.kind(), unsafe { sqlite_res.unwrap_err_unchecked().kind() });
-				assert!(test_res.is_err());
-				assert_eq!(e.kind(), unsafe { test_res.unwrap_err_unchecked().kind() });
-				Err(e)
-			}
-		}
-	}
-
-	fn write(
-		&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8],
-	) -> std::io::Result<()> {
-		let _guard = self.serializer.write().unwrap();
-		let fs_res = self.fs_store.write(primary_namespace, secondary_namespace, key, buf);
-		let sqlite_res = self.sqlite_store.write(primary_namespace, secondary_namespace, key, buf);
-		let test_res = self.test_store.write(primary_namespace, secondary_namespace, key, buf);
-
-		assert!(self
-			.do_list(primary_namespace, secondary_namespace)
-			.unwrap()
-			.contains(&key.to_string()));
-
-		match fs_res {
-			Ok(()) => {
-				assert!(sqlite_res.is_ok());
-				assert!(test_res.is_ok());
-				Ok(())
-			}
-			Err(e) => {
-				assert!(sqlite_res.is_err());
-				assert!(test_res.is_err());
-				Err(e)
-			}
-		}
-	}
-
-	fn remove(
-		&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool,
-	) -> std::io::Result<()> {
-		let _guard = self.serializer.write().unwrap();
-		let fs_res = self.fs_store.remove(primary_namespace, secondary_namespace, key, lazy);
-		let sqlite_res =
-			self.sqlite_store.remove(primary_namespace, secondary_namespace, key, lazy);
-		let test_res = self.test_store.remove(primary_namespace, secondary_namespace, key, lazy);
-
-		assert!(!self
-			.do_list(primary_namespace, secondary_namespace)
-			.unwrap()
-			.contains(&key.to_string()));
-
-		match fs_res {
-			Ok(()) => {
-				assert!(sqlite_res.is_ok());
-				assert!(test_res.is_ok());
-				Ok(())
-			}
-			Err(e) => {
-				assert!(sqlite_res.is_err());
-				assert!(test_res.is_err());
-				Err(e)
-			}
-		}
-	}
-
-	fn list(
-		&self, primary_namespace: &str, secondary_namespace: &str,
-	) -> std::io::Result<Vec<String>> {
-		let _guard = self.serializer.read().unwrap();
-		self.do_list(primary_namespace, secondary_namespace)
-	}
-}

src/lib.rs

@@ -85,8 +85,6 @@ mod logger;
 mod payment_store;
 mod peer_store;
 mod sweep;
-#[cfg(test)]
-mod test;
 mod tx_broadcaster;
 mod types;
 #[cfg(feature = "uniffi")]

src/payment_store.rs

@@ -219,8 +219,7 @@ where
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::test::utils::TestLogger;
-	use lightning::util::test_utils::TestStore;
+	use lightning::util::test_utils::{TestLogger, TestStore};
 	use std::sync::Arc;
 
 	#[test]

src/peer_store.rs

@@ -143,9 +143,7 @@ impl_writeable_tlv_based!(PeerInfo, {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::test::utils::TestLogger;
-
-	use lightning::util::test_utils::TestStore;
+	use lightning::util::test_utils::{TestLogger, TestStore};
 
 	use std::str::FromStr;
 	use std::sync::Arc;