Postgresql-setup script updates

docker-compose*.yml: Bump node version to 10.1.4 and dbsync version to 13.6.0.4
postgresql-setup.sh:
- Parse psql output using csv instead of manipulating white chars or table seperators
- Use strict checks for ${PGDATABASE} as there can be another database with a prefix/suffix attached
- When creating/restoring a snapshot, existence of ledger_file should be optional, as there are dbsync config options that don't require lstate files, accordingly - make presence of ledger-state-file optional
- When restoring a snapshot (eg: across postgres versions), there will be cases where an informational error may be expected. The presence of exit-on-error prevents use of these snapshots, especially when run from docker created by nix-ops (where it isnt as straightforward to override scripts)

Author: rdlrt

docker-compose.example.yml

@@ -36,7 +36,7 @@ services:
         max-file: "10"
 
   cardano-node:
-    image: ghcr.io/intersectmbo/cardano-node:9.0.0
+    image: ghcr.io/intersectmbo/cardano-node:10.1.4
     environment:
       - NETWORK=${NETWORK:-mainnet}
     networks:
@@ -59,7 +59,7 @@ services:
         max-file: "10"
 
   cardano-db-sync:
-    image: ghcr.io/intersectmbo/cardano-db-sync:13.3.0.0
+    image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4
     environment:
       - NETWORK=${NETWORK:-mainnet}
       - POSTGRES_HOST=postgres

docker-compose.yml

@@ -33,7 +33,7 @@ services:
         max-file: "10"
 
   cardano-node:
-    image: ghcr.io/intersectmbo/cardano-node:10.1.3
+    image: ghcr.io/intersectmbo/cardano-node:10.1.4
     environment:
       - NETWORK=${NETWORK:-mainnet}
     volumes:

scripts/postgresql-setup.sh

@@ -98,7 +98,7 @@ function check_connect_as_user {
 
 function check_db_exists {
 	set +e
-	count=$(psql -l "${PGDATABASE}" | grep -c "${PGDATABASE} ")
+	count=$(psql -t -l --csv "${PGDATABASE}" | grep -c "^${PGDATABASE},")
 	if test "${count}" -lt 1 ; then
 		echo
 		echo "Error : No '${PGDATABASE}' database."
@@ -108,7 +108,7 @@ function check_db_exists {
 		echo
 		exit 1
 		fi
-	count=$(psql -l "${PGDATABASE}" | grep "${PGDATABASE} " | cut -d \| -f 3 | grep -c UTF8)
+	count=$(psql -t -l --csv "${PGDATABASE}" | grep "^${PGDATABASE}," | grep -c UTF8)
 	if test "${count}" -ne 1 ; then
 		echo
 		echo "Error : '${PGDATABASE}' database exists, but is not UTF8."
@@ -164,19 +164,21 @@ function create_snapshot {
 	tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
 	echo $"Working directory: ${tmp_dir}"
 	pg_dump --no-owner --schema=public --jobs="${numcores}" "${PGDATABASE}" --format=directory --file="${tmp_dir}/db/"
-	lstate_gz_file=$(basename "${ledger_file}").gz
-	gzip --to-stdout "${ledger_file}" > "${tmp_dir}/$(basename "${ledger_file}").gz"
-	tree "${tmp_dir}"
-	# Use plain tar here because the database dump files and the ledger state file are already gzipped.
-	tar cvf - --directory "${tmp_dir}" "db" "${lstate_gz_file}" | tee "${tgz_file}.tmp" \
-		| sha256sum | head -c 64 | sed -e "s/$/  ${tgz_file}\n/" > "${tgz_file}.sha256sum"
+	if [ -n "${ledger_file}" ]; then
+	  lstate_gz_file=$(basename "${ledger_file}").gz
+	  gzip --to-stdout "${ledger_file}" > "${tmp_dir}/${lstate_gz_file}"
+	fi
+	# Use plain tar here because the database dump files and the ledger state file are already gzipped. Disable Shellcheck SC2046 to avoid empty '' getting added while quoting
+	# shellcheck disable=SC2046
+	tar cvf - --directory "${tmp_dir}" "db" $( [ -n "${lstate_gz_file:-}" ] && [ -f "/${tmp_dir}/${lstate_gz_file}" ] && echo "${lstate_gz_file}" ) | tee "${tgz_file}.tmp" | \
+	  sha256sum | head -c 64 | sed -e "s/$/  ${tgz_file}\n/" > "${tgz_file}.sha256sum"
 	mv "${tgz_file}.tmp" "${tgz_file}"
 	rm "${recursive}" "${force}" "${tmp_dir}"
 	if test "$(tar "${test}" --file "${tgz_file}")" ; then
 	  echo "Tar reports the snapshot file as being corrupt."
 	  echo "It is not safe to drop the database and restore using this file."
 	  exit 1
-	  fi
+	fi
 	echo "Created ${tgz_file} + .sha256sum"
 }
 
@@ -185,24 +187,25 @@ function restore_snapshot {
 	if test "${file_count}" -gt 0 ; then
 	  echo "Ledger state directory ($2) is not empty. Please empty it and then retry."
 	  exit 1
-	  fi
+	fi
 	tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
 	tar xvf "$1" --directory "$tmp_dir"
 	if test -d "${tmp_dir}/db/" ; then
 	  # New pg_dump format
-	  lstate_gz_file=$(find "${tmp_dir}/" -iname "*.lstate.gz")
-	  lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
-	  gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
+	  if [ -n "${lstate_gz_file:-}" ] ; then
+	    lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
+	    gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
+	  fi
 
-          # Important: specify --schema=public below to skip over `create schema public`
-          # statement generated by pg_dump
+	  # Important: specify --schema=public below to skip over `create schema public`
+	  # statement generated by pg_dump
 	  pg_restore \
 	    --schema=public \
 	    --jobs="${numcores}" \
 	    --format=directory \
 	    --dbname="${PGDATABASE}" \
-            --no-owner \
-	    --exit-on-error \
+	    --no-owner \
+	    "$( [ -z "${SKIP_RESTORE_ERROR}" ] && echo " --exit-on-error" )" \
 	    "${tmp_dir}/db/"
 	else
 	  # Old snapshot format produced by this script
@@ -228,7 +231,7 @@ function usage_exit {
 	echo "    $progname --dump-schema       - Dump the schema of the database."
 	echo
 	echo "    - Create a db-sync state snapshot"
-	echo "      $progname --create-snapshot <snapshot-file> <ledger-state-file>"
+	echo "      $progname --create-snapshot <snapshot-file> [<ledger-state-file>]"
 	echo
 	echo "    - Restore a db-sync state snapshot."
 	echo "      $progname --restore-snapshot <snapshot-file> <ledger-state-dir>"
@@ -300,23 +303,19 @@ case "${1:-""}" in
 	--create-snapshot)
 		check_pgpass_file
 		check_db_exists
-		if test $# -ne 3 ; then
-		  echo "Expecting exactly 2 more arguments, the snapshot file name template and the ledger state file."
+		if test $# -lt 2 ; then
+		  echo "Expecting the snapshot file name (without extension) and optionally the ledger state file as arguments."
 		  exit 1
-		  fi
+		fi
 		if test -z "$2" ; then
-		  echo "Second argument should be the snapshot file name template."
+		  echo "Second argument should be the snapshot file name (without extension)."
 		  exit 1
-		  fi
-		if test -z "$3" ; then
-		  echo "Third argument should be the ledger state file."
-		  exit 1
-		  fi
-		if test -d "$3" ; then
-		  echo "Third argument is a directory and expecting a file."
+		fi
+		if test -n "${3:-}" && test -d "${3}" ; then
+		  echo "Third argument provided is a directory but expecting a file."
 		  exit 1
-		  fi
-		create_snapshot "$2" "$3"
+		fi
+		create_snapshot "$2" "${3:-}"
 		;;
 	--restore-snapshot)
 		check_pgpass_file
@@ -334,15 +333,11 @@ case "${1:-""}" in
 		  echo "Second argument should be the snapshot file."
 		  exit 1
 		  fi
-		if test -z "$3" ; then
-		  echo "Third argument should be the ledger state directory."
-		  exit 1
-		  fi
-		if test -f "$3" ; then
+		if test -n "${3:-}" && test -f "${3}"; then
 		  echo "Third argument is a file and expecting a directory."
 		  exit 1
 		  fi
-		restore_snapshot "$2" "$3"
+		restore_snapshot "$2" "${3:-}"
 		;;
 	*)
 		usage_exit