From d5f1b79012866ff10b855f27ce40c9277f8f6413 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 17 Jun 2020 16:22:21 +0200 Subject: [PATCH 1/3] add way to import database from a folder --- packages/postgres-database/Makefile | 36 ++++++++-- .../scripts/copy_database_volume.sh | 65 +++++++++++++++++-- 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/packages/postgres-database/Makefile b/packages/postgres-database/Makefile index 569261dddb4..8cafd06c062 100644 --- a/packages/postgres-database/Makefile +++ b/packages/postgres-database/Makefile @@ -22,21 +22,47 @@ tests: ## runs unit tests .PHONY: import-db -import-db: scripts/copy_database_volume.sh guard-SOURCE_HOST guard-SOURCE_DATA_VOLUME guard-TARGET_DATA_VOLUME ## copy volume $(SOURCE_DATA_VOLUME) from $(SOURCE_HOST) into local $(TARGET_DATA_VOLUME) - ./scripts/copy_database_volume.sh - +#guard-SOURCE_HOST guard-SOURCE_DATA_VOLUME guard-TARGET_DATA_VOLUME +import-db-from-docker-volume import-db-from-folder: scripts/copy_database_volume.sh ## copy postgresql data from remote $host from $host_volume or $host_folder to a local $local_volume docker volume + @:$(if $(findstring -from-folder,$@),\ + $(call check_defined, host host_folder local_volume, please define this variable when calling $@), \ + $(call check_defined, host host_volume local_volume, please define this variable when calling $@)) + ./scripts/copy_database_volume.sh \ + --host $(host) \ + $(if $(findstring -from-folder, $@),--folder $(host_folder),--volume $(host_volume)) \ + --target $(local_volume) + +# Check that given variables are set and all have non-empty values, +# die with an error otherwise. +# +# Params: +# 1. Variable name(s) to test. +# 2. (optional) Error message to print. guard-%: @ if [ "${${*}}" = "" ]; then \ echo "Environment variable $* not set"; \ exit 1; \ fi +# Check that given variables are set and all have non-empty values, +# die with an error otherwise. +# +# Params: +# 1. Variable name(s) to test. +# 2. (optional) Error message to print. +check_defined = \ + $(strip $(foreach 1,$1, \ + $(call __check_defined,$1,$(strip $(value 2))))) +__check_defined = \ + $(if $(value $1),, \ + $(error Undefined $1$(if $2, ($2)))) + .PHONY: setup-prod -export POSTGRES_DATA_VOLUME = $(TARGET_DATA_VOLUME) -setup-prod: install-dev up-prod ## sets up a database using an external postgres volume to test migration +setup-prod: guard-POSTGRES_DATA_VOLUME install-dev up-prod ## sets up a database using an external postgres volume defined as $POSTGRES_DATA_VOLUME to test migration # discovering sc-pg --help @echo "To test migration, sc-pg discover -u USER -p PASSWORD, then sc-pg upgrade" + # adminer: http://127.0.0.1:18080/?pgsql=postgres&ns=public .PHONY: setup-commit setup-commit: install-dev up-pg ## sets up a database to create a new commit into migration history diff --git a/packages/postgres-database/scripts/copy_database_volume.sh b/packages/postgres-database/scripts/copy_database_volume.sh index 9b3f58af810..681ce459533 100755 --- a/packages/postgres-database/scripts/copy_database_volume.sh +++ b/packages/postgres-database/scripts/copy_database_volume.sh @@ -1,10 +1,67 @@ #!/bin/sh + +usage() +{ + echo "usage: copy_database_volume.sh [[[-h host ] [[-f folder] | [-v volume]] [-t target]] | [-h]]" +} + + +if [ $# -eq 0 ]; then + usage + exit 1 +fi + + +while [ "$1" != "" ]; do + case $1 in + -h | --host ) shift + host=$1 + ;; + -f | --folder ) shift + folder=$1 + ;; + -v | --volume ) shift + volume=$1 + ;; + -t | --target ) shift + target=$1 + ;; + -h | --help ) usage + exit + ;; + * ) usage + exit 1 + esac + shift +done + +if [ -z $host ] || [ -z $target ] || ([ -z $folder ] && [ -z $volume ]); then + usage + exit 1 +fi + +if [ ! -z $folder ] && [ ! -z $volume ]; then + echo "cannot use both --folder and --volume arguments" + usage + exit 1 +fi + set -o errexit -set -o nounset +# set -o nounset IFS=$(printf '\n\t') -ssh "${SOURCE_HOST}" \ - "docker run --rm -v ${SOURCE_DATA_VOLUME}:/from alpine ash -c 'cd /from ; tar -cf - . '" \ + +if [ ! -z $folder ]; then + #folder mode + ssh $host \ + "cd $folder; tar -cf - . " \ + | \ + docker run --rm -i -v "$target":/to alpine ash -c "cd /to ; tar -xpvf - " +else + #docker mode + ssh $host \ + "docker run --rm -v $volume:/from alpine ash -c 'cd /from ; tar -cf - . '" \ | \ - docker run --rm -i -v "${TARGET_DATA_VOLUME}":/to alpine ash -c "cd /to ; tar -xpvf - " \ No newline at end of file + docker run --rm -i -v "$target":/to alpine ash -c "cd /to ; tar -xpvf - " +fi From ed0b7b63188c80f34e67216b146b5497e55ac0cb Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 17 Jun 2020 17:01:06 +0200 Subject: [PATCH 2/3] script to upgrade projects that uses a notebook 2.13.0 to use a 2.13.1 notebook --- .../scripts/migrate_notebook_2_13_0_to_2_13_1.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 packages/postgres-database/scripts/migrate_notebook_2_13_0_to_2_13_1.sql diff --git a/packages/postgres-database/scripts/migrate_notebook_2_13_0_to_2_13_1.sql b/packages/postgres-database/scripts/migrate_notebook_2_13_0_to_2_13_1.sql new file mode 100644 index 00000000000..9e72703cb4b --- /dev/null +++ b/packages/postgres-database/scripts/migrate_notebook_2_13_0_to_2_13_1.sql @@ -0,0 +1,15 @@ +-- select the projects with the services to update (replace the regexp accordingly. NOTE: % is equivalent to .* in SQL) +-- SELECT workbench +-- FROM projects +-- WHERE workbench::text SIMILAR TO '%-notebook", "version": "2.13.0"%' +-- replace the regexp in here in order to +UPDATE projects +SET workbench = ( + regexp_replace( + workbench::text, + '-notebook", "version": "2.13.0"', + '-notebook", "version": "2.13.1"', + 'g' + )::json + ) +WHERE workbench::text SIMILAR TO '%-notebook", "version": "2.13.0"%' From 18897233210e5b0162285996abefd1d54d4bd32a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 17 Jun 2020 17:18:29 +0200 Subject: [PATCH 3/3] in case of failing rights this will fail right away --- packages/postgres-database/scripts/copy_database_volume.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/scripts/copy_database_volume.sh b/packages/postgres-database/scripts/copy_database_volume.sh index 681ce459533..227c1e46d4e 100755 --- a/packages/postgres-database/scripts/copy_database_volume.sh +++ b/packages/postgres-database/scripts/copy_database_volume.sh @@ -55,7 +55,7 @@ IFS=$(printf '\n\t') if [ ! -z $folder ]; then #folder mode ssh $host \ - "cd $folder; tar -cf - . " \ + "tar -cf - $folder " \ | \ docker run --rm -i -v "$target":/to alpine ash -c "cd /to ; tar -xpvf - " else