Skip to content

Postgresql-setup script updates #1935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 5, 2025
Merged

Postgresql-setup script updates #1935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docker-compose.example.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ services:
max-file: "10"

cardano-node:
image: ghcr.io/intersectmbo/cardano-node:9.0.0
image: ghcr.io/intersectmbo/cardano-node:10.1.4
environment:
- NETWORK=${NETWORK:-mainnet}
networks:
Expand All @@ -59,7 +59,7 @@ services:
max-file: "10"

cardano-db-sync:
image: ghcr.io/intersectmbo/cardano-db-sync:13.3.0.0
image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4
environment:
- NETWORK=${NETWORK:-mainnet}
- POSTGRES_HOST=postgres
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ services:
max-file: "10"

cardano-node:
image: ghcr.io/intersectmbo/cardano-node:10.1.3
image: ghcr.io/intersectmbo/cardano-node:10.1.4
environment:
- NETWORK=${NETWORK:-mainnet}
volumes:
Expand Down
70 changes: 33 additions & 37 deletions scripts/postgresql-setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ function check_connect_as_user {

function check_db_exists {
set +e
count=$(psql -l "${PGDATABASE}" | grep -c "${PGDATABASE} ")
count=$(psql -t -l --csv "${PGDATABASE}" | grep -c "^${PGDATABASE},")
if test "${count}" -lt 1 ; then
echo
echo "Error : No '${PGDATABASE}' database."
Expand All @@ -108,7 +108,7 @@ function check_db_exists {
echo
exit 1
fi
count=$(psql -l "${PGDATABASE}" | grep "${PGDATABASE} " | cut -d \| -f 3 | grep -c UTF8)
count=$(psql -t -l --csv "${PGDATABASE}" | grep "^${PGDATABASE}," | grep -c UTF8)
if test "${count}" -ne 1 ; then
echo
echo "Error : '${PGDATABASE}' database exists, but is not UTF8."
Expand Down Expand Up @@ -164,19 +164,21 @@ function create_snapshot {
tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
echo $"Working directory: ${tmp_dir}"
pg_dump --no-owner --schema=public --jobs="${numcores}" "${PGDATABASE}" --format=directory --file="${tmp_dir}/db/"
lstate_gz_file=$(basename "${ledger_file}").gz
gzip --to-stdout "${ledger_file}" > "${tmp_dir}/$(basename "${ledger_file}").gz"
tree "${tmp_dir}"
# Use plain tar here because the database dump files and the ledger state file are already gzipped.
tar cvf - --directory "${tmp_dir}" "db" "${lstate_gz_file}" | tee "${tgz_file}.tmp" \
| sha256sum | head -c 64 | sed -e "s/$/ ${tgz_file}\n/" > "${tgz_file}.sha256sum"
if [ -n "${ledger_file}" ]; then
lstate_gz_file=$(basename "${ledger_file}").gz
gzip --to-stdout "${ledger_file}" > "${tmp_dir}/${lstate_gz_file}"
fi
# Use plain tar here because the database dump files and the ledger state file are already gzipped. Disable Shellcheck SC2046 to avoid empty '' getting added while quoting
# shellcheck disable=SC2046
tar cvf - --directory "${tmp_dir}" "db" $( [ -n "${lstate_gz_file:-}" ] && [ -f "/${tmp_dir}/${lstate_gz_file}" ] && echo "${lstate_gz_file}" ) | tee "${tgz_file}.tmp" | \
sha256sum | head -c 64 | sed -e "s/$/ ${tgz_file}\n/" > "${tgz_file}.sha256sum"
mv "${tgz_file}.tmp" "${tgz_file}"
rm "${recursive}" "${force}" "${tmp_dir}"
if test "$(tar "${test}" --file "${tgz_file}")" ; then
echo "Tar reports the snapshot file as being corrupt."
echo "It is not safe to drop the database and restore using this file."
exit 1
fi
fi
echo "Created ${tgz_file} + .sha256sum"
}

Expand All @@ -185,24 +187,26 @@ function restore_snapshot {
if test "${file_count}" -gt 0 ; then
echo "Ledger state directory ($2) is not empty. Please empty it and then retry."
exit 1
fi
fi
tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
tar xvf "$1" --directory "$tmp_dir"
if test -d "${tmp_dir}/db/" ; then
# New pg_dump format
lstate_gz_file=$(find "${tmp_dir}/" -iname "*.lstate.gz")
lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
if [ -n "${lstate_gz_file:-}" ] ; then
lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
fi

# Important: specify --schema=public below to skip over `create schema public`
# statement generated by pg_dump
# Important: specify --schema=public below to skip over `create schema public`
# statement generated by pg_dump
# shellcheck disable=SC2046
pg_restore \
--schema=public \
--jobs="${numcores}" \
--format=directory \
--dbname="${PGDATABASE}" \
--no-owner \
--exit-on-error \
--no-owner \
$( [ -z "${SKIP_RESTORE_ERROR:-}" ] && echo "--exit-on-error" ) \
"${tmp_dir}/db/"
else
# Old snapshot format produced by this script
Expand All @@ -228,7 +232,7 @@ function usage_exit {
echo " $progname --dump-schema - Dump the schema of the database."
echo
echo " - Create a db-sync state snapshot"
echo " $progname --create-snapshot <snapshot-file> <ledger-state-file>"
echo " $progname --create-snapshot <snapshot-file> [<ledger-state-file>]"
echo
echo " - Restore a db-sync state snapshot."
echo " $progname --restore-snapshot <snapshot-file> <ledger-state-dir>"
Expand Down Expand Up @@ -300,23 +304,19 @@ case "${1:-""}" in
--create-snapshot)
check_pgpass_file
check_db_exists
if test $# -ne 3 ; then
echo "Expecting exactly 2 more arguments, the snapshot file name template and the ledger state file."
if test $# -lt 2 ; then
echo "Expecting the snapshot file name (without extension) and optionally the ledger state file as arguments."
exit 1
fi
fi
if test -z "$2" ; then
echo "Second argument should be the snapshot file name template."
echo "Second argument should be the snapshot file name (without extension)."
exit 1
fi
if test -z "$3" ; then
echo "Third argument should be the ledger state file."
exit 1
fi
if test -d "$3" ; then
echo "Third argument is a directory and expecting a file."
fi
if test -n "${3:-}" && test -d "${3}" ; then
echo "Third argument provided is a directory but expecting a file."
exit 1
fi
create_snapshot "$2" "$3"
fi
create_snapshot "$2" "${3:-}"
;;
--restore-snapshot)
check_pgpass_file
Expand All @@ -334,15 +334,11 @@ case "${1:-""}" in
echo "Second argument should be the snapshot file."
exit 1
fi
if test -z "$3" ; then
echo "Third argument should be the ledger state directory."
exit 1
fi
if test -f "$3" ; then
if test -n "${3:-}" && test -f "${3}"; then
echo "Third argument is a file and expecting a directory."
exit 1
fi
restore_snapshot "$2" "$3"
restore_snapshot "$2" "${3:-}"
;;
*)
usage_exit
Expand Down
Loading