Sync test script

You can test whether the branch contains breaking changes from master using this script. This script simply tries to sync from scratch on both branches and tries to continue to sync on the other branch. Although it doesn't automatically detect state root mismatches, you can easily see the progress stalls.

#/bin/bash

## USAGE
## ./cross-sync.sh master feature/blah ./db-synctest 192.168.100.123:3485 192.168.100.123:8080

BASE=$1
REMOTE=$2
DB_PATH=$3
BOOTSTRAP=$4
JSON_RPC=$5

function best_block_number() {
	local HOST=$1

	local RESP=`curl --retry 5 --retry-connrefused \
		-s -H 'Content-Type: application/json' \
		-d '{"jsonrpc": "2.0", "method": "chain_getBestBlockNumber", "params": [], "id": null}' \
		$HOST`
	echo $RESP | sed -n 's/.*"result":\([0-9]*\).*/\1/p'
}

function upstream_block_number() { best_block_number "$JSON_RPC"; }

function current_block_number() { best_block_number "http://localhost:8080"; }

function sync_to() {
	local TARGET_BN=$1
	local DB_PATH=$2
	local BRANCH=$3
	local WAIT_MORE=$4
	
	git checkout $BRANCH
	cargo build --release

	ENABLE_DELEGATIONS=true \
	ENABLE_ORDER=true \
	RUST_LOG=info,mio=warn,tokio=warn,hyper=warn,rpc=info \
	./target/release/codechain -c corgi --jsonrpc-port 8080 --port 3485 \
		--no-tx-relay --enable-devel-api --no-miner --no-discovery \
		--bootstrap-addresses $BOOTSTRAP \
		--db-path "$DB_PATH" &
	CODECHAIN=$!
	trap "kill -INT $CODECHAIN; wait $CODECHAIN; exit" SIGINT
	sleep 2
	while true; do
		BN=`current_block_number`
		if [ "$BN" -ge "$TARGET_BN" ]; then 
			break
		fi
		echo "$BRANCH-$DB_PATH $BN/$TARGET_BN" >> ./cross-sync.log
		sleep 1
	done
	sleep $WAIT_MORE
	kill -INT $CODECHAIN
	wait $CODECHAIN
	trap 'exit -1' SIGINT
}

function run(){
	local DB_PATH=$1
	local SCRATCH=$2
	local CONTINUE=$3

	# sccache makes 'error reading compile response from server' error
	unset RUSTC_WRAPPER

	# Sync from scratch with branch SCRATCH
	sync_to `upstream_block_number` "$DB_PATH" "$SCRATCH" 0

	# Continue to sync with branch CONTINUE
	sync_to `upstream_block_number` "$DB_PATH" "$CONTINUE" 60
}

set -ex
echo "" > ./cross-sync.log
run "$DB_PATH-base-remote" "$BASE" "$REMOTE"
run "$DB_PATH-remote-base" "$REMOTE" "$BASE"

If your JSONRPC is behind the ssh, you can try forwarding

ssh ssh -o 'ControlPath ./jsonrpc.ctrl' -fNTM -L 9000:localhost:8080 $SSH_JSON_RPC
bash cross-sync.sh master feature/blah ./db-synctest 192.168.100.123:3485 localhost:9000
ssh -o 'ControlPath ./jsonrpc.ctrl' -TO exit $SSH_JSON_RPC