Skip to content

Commit 1dc6753

Browse files
pcncpcnc
authored
feat: automate pg_upgrade testing (#695)
1 parent 7a75858 commit 1dc6753

File tree

14 files changed

+726
-32
lines changed

14 files changed

+726
-32
lines changed

‎.github/workflows/publish-pgupgrade-scripts.yml

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ permissions:
1313
id-token: write
1414

1515
jobs:
16-
build:
16+
publish-staging:
1717
runs-on: ubuntu-latest
1818

1919
steps:
@@ -42,6 +42,36 @@ jobs:
4242
run: |
4343
aws s3 cp /tmp/pg_upgrade_scripts.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz
4444
45+
- name: Slack Notification on Failure
46+
if: ${{ failure() }}
47+
uses: rtCamp/action-slack-notify@v2
48+
env:
49+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
50+
SLACK_USERNAME: 'gha-failures-notifier'
51+
SLACK_COLOR: 'danger'
52+
SLACK_MESSAGE: 'Publishing pg_upgrade scripts failed'
53+
SLACK_FOOTER: ''
54+
55+
publish-prod:
56+
runs-on: ubuntu-latest
57+
if: github.ref_name == 'develop'
58+
59+
steps:
60+
- name: Checkout Repo
61+
uses: actions/checkout@v3
62+
63+
- name: Grab release version
64+
id: process_release_version
65+
run: |
66+
VERSION=$(grep 'postgres-version' common.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
67+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
68+
69+
- name: Create a tarball containing pg_upgrade scripts
70+
run: |
71+
mkdir -p /tmp/pg_upgrade_scripts
72+
cp -r ansible/files/admin_api_scripts/pg_upgrade_scripts/* /tmp/pg_upgrade_scripts
73+
tar -czvf /tmp/pg_upgrade_scripts.tar.gz -C /tmp/ pg_upgrade_scripts
74+
4575
- name: configure aws credentials - prod
4676
uses: aws-actions/configure-aws-credentials@v1
4777
with:

‎.github/workflows/test-pg-upgrade.yml

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
name: Test pg_upgrade
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
- pcnc/auto-pg_upgrade-testing
8+
workflow_dispatch:
9+
10+
permissions:
11+
id-token: write
12+
13+
jobs:
14+
test:
15+
strategy:
16+
matrix:
17+
base_pg_version:
18+
- 15.1.1.60
19+
runs-on: arm-runner
20+
timeout-minutes: 30
21+
defaults:
22+
run:
23+
working-directory: ./tests/pg_upgrade
24+
env:
25+
PGPORT: 5478
26+
PGPASSWORD: postgres
27+
PGDATABASE: postgres
28+
PGUSER: supabase_admin
29+
PGHOST: localhost
30+
PG_MAJOR_VERSION: 15
31+
IS_CI: true
32+
container: pg_upgrade_test
33+
steps:
34+
- uses: actions/checkout@v3
35+
36+
- name: Grab release version
37+
id: process_release_version
38+
working-directory: ./
39+
run: |
40+
VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common.vars.pkr.hcl)
41+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
42+
43+
- name: configure aws credentials - staging
44+
uses: aws-actions/configure-aws-credentials@v1
45+
with:
46+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
47+
aws-region: "us-east-1"
48+
49+
- name: Download pg_upgrade_scripts and binaries
50+
run: |
51+
aws s3 cp s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz scripts/pg_upgrade_scripts.tar.gz
52+
aws s3 cp s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz scripts/pg_upgrade_bin.tar.gz
53+
54+
- run: docker context create builders
55+
- uses: docker/setup-buildx-action@v2
56+
with:
57+
endpoint: builders
58+
driver-opts: image=moby/buildkit:v0.11.6
59+
buildkitd-flags: --debug
60+
61+
- name: Start Postgres
62+
run: |
63+
docker rm -f "$container" || true
64+
docker run --name "$container" --env-file .env \
65+
-v "$(pwd)/scripts:/tmp/upgrade" \
66+
--entrypoint "/tmp/upgrade/entrypoint.sh" -d \
67+
-p ${PGPORT}:5432 \
68+
"supabase/postgres:${{ matrix.base_pg_version }}"
69+
70+
- name: Install psql
71+
run: |
72+
sudo apt update
73+
sudo apt install -y --no-install-recommends postgresql-client
74+
75+
- name: Install pg_prove
76+
run: |
77+
sudo apt-get update
78+
sudo apt-get install -y --no-install-recommends perl cpanminus
79+
sudo cpanm -n App::cpanminus
80+
sudo cpanm -n TAP::Parser::SourceHandler::pgTAP
81+
env:
82+
SHELL: /bin/bash
83+
PERL_MM_USE_DEFAULT: 1
84+
PERL_MM_NONINTERACTIVE: 1
85+
86+
- name: Wait for healthy database
87+
run: |
88+
count=0
89+
while ! docker exec "$container" bash -c "pg_isready"; do
90+
count=$((count + 1))
91+
if [ $count -ge "$retries" ]; then
92+
echo "Retry $count/$retries exited $exit, no more retries left."
93+
docker logs "$container"
94+
docker rm -f "$container"
95+
exit 1
96+
fi
97+
done
98+
env:
99+
retries: 20
100+
101+
- name: Run migrations
102+
run: docker exec "$container" bash -c "/docker-entrypoint-initdb.d/migrate.sh > /tmp/migrate.log 2>&1"
103+
104+
- name: Run initial tests
105+
run: pg_prove "../../migrations/tests/test.sql"
106+
env:
107+
PERL5LIB: /usr/local/lib/perl5
108+
109+
- name: Apply pre-upgrade fixtures
110+
run: |
111+
psql -f "./tests/98-data-fixtures.sql"
112+
psql -f "./tests/99-fixtures.sql"
113+
114+
- name: Initiate upgrade
115+
run: docker exec "$container" bash -c '/tmp/upgrade/pg_upgrade_scripts/initiate.sh "$PG_MAJOR_VERSION"; exit $?'
116+
117+
- name: Complete pg_upgrade
118+
run: docker exec pg_upgrade_test bash -c '/tmp/upgrade/pg_upgrade_scripts/complete.sh; exit $?'
119+
120+
- name: Run post-upgrade tests
121+
run: |
122+
pg_prove tests/01-schema.sql
123+
pg_prove tests/02-data.sql
124+
pg_prove tests/03-settings.sql
125+
126+
- name: Clean up container
127+
if: ${{ always() }}
128+
continue-on-error: true
129+
run: docker rm -f "$container" || true

‎ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,25 @@ function retry {
6161
done
6262
return 0
6363
}
64+
65+
CI_stop_postgres() {
66+
BINDIR=$(pg_config --bindir)
67+
ARG=${1:-""}
68+
69+
if [ "$ARG" = "--new-bin" ]; then
70+
BINDIR="/tmp/pg_upgrade_bin/$PG_MAJOR_VERSION/bin"
71+
fi
72+
73+
su postgres -c "$BINDIR/pg_ctl stop -o '-c config_file=/etc/postgresql/postgresql.conf' -l /tmp/postgres.log"
74+
}
75+
76+
CI_start_postgres() {
77+
BINDIR=$(pg_config --bindir)
78+
ARG=${1:-""}
79+
80+
if [ "$ARG" = "--new-bin" ]; then
81+
BINDIR="/tmp/pg_upgrade_bin/$PG_MAJOR_VERSION/bin"
82+
fi
83+
84+
su postgres -c "$BINDIR/pg_ctl start -o '-c config_file=/etc/postgresql/postgresql.conf' -l /tmp/postgres.log"
85+
}

‎ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ SCRIPT_DIR=$(dirname -- "$0";)
1111
# shellcheck disable=SC1091
1212
source "$SCRIPT_DIR/common.sh"
1313

14+
IS_CI=${IS_CI:-}
1415
LOG_FILE="/var/log/pg-upgrade-complete.log"
1516

1617
function cleanup {
@@ -81,14 +82,22 @@ function complete_pg_upgrade {
8182
echo "running" > /tmp/pg-upgrade-status
8283

8384
echo "1. Mounting data disk"
84-
retry 8 mount -a -v
85+
if [ -z "$IS_CI" ]; then
86+
retry 8 mount -a -v
87+
else
88+
echo "Skipping mount -a -v"
89+
fi
8590

8691
# copying custom configurations
8792
echo "2. Copying custom configurations"
8893
retry 3 copy_configs
8994

9095
echo "3. Starting postgresql"
91-
retry 3 service postgresql start
96+
if [ -z "$IS_CI" ]; then
97+
retry 3 service postgresql start
98+
else
99+
CI_start_postgres --new-bin
100+
fi
92101

93102
echo "4. Running generated SQL files"
94103
retry 3 run_generated_sql
@@ -104,11 +113,16 @@ function complete_pg_upgrade {
104113
sleep 5
105114

106115
echo "5. Restarting postgresql"
107-
retry 3 service postgresql restart
108-
109-
echo "5.1. Restarting gotrue and postgrest"
110-
retry 3 service gotrue restart
111-
retry 3 service postgrest restart
116+
if [ -z "$IS_CI" ]; then
117+
retry 3 service postgresql restart
118+
119+
echo "5.1. Restarting gotrue and postgrest"
120+
retry 3 service gotrue restart
121+
retry 3 service postgrest restart
122+
else
123+
retry 3 CI_stop_postgres || true
124+
retry 3 CI_start_postgres
125+
fi
112126

113127
echo "6. Starting vacuum analyze"
114128
retry 3 start_vacuum_analyze
@@ -137,7 +151,10 @@ function apply_auth_scheme_updates {
137151
if [ "$PASSWORD_ENCRYPTION_SETTING" = "md5" ]; then
138152
run_sql -c "ALTER SYSTEM SET password_encryption TO 'scram-sha-256';"
139153
run_sql -c "SELECT pg_reload_conf();"
140-
run_sql -f /etc/postgresql.schema.sql
154+
155+
if [ -z "$IS_CI" ]; then
156+
run_sql -f /etc/postgresql.schema.sql
157+
fi
141158
fi
142159
}
143160

@@ -149,5 +166,16 @@ function start_vacuum_analyze {
149166

150167
trap cleanup ERR
151168

169+
if [ -z "$IS_CI" ]; then
170+
complete_pg_upgrade >> $LOG_FILE 2>&1 &
171+
else
172+
CI_stop_postgres || true
173+
174+
rm -f /tmp/pg-upgrade-status
175+
mv /data_migration /data
176+
177+
rm -rf /var/lib/postgresql/data
178+
ln -s /data/pgdata /var/lib/postgresql/data
152179

153-
complete_pg_upgrade >> $LOG_FILE 2>&1 &
180+
complete_pg_upgrade
181+
fi

0 commit comments

Comments
 (0)