Skip to content

Commit d1cb4ca

Browse files
samrosesamrose
authored
Sam/wal g 2 and 3 (#1485)
* feature: wal-g v 3 and v 2 * chore: install, symlink * chore: bump versions to test wal-g * chore: build wal-g in ci and copy to binary cache on success * chore: release wal-g 3 * chore: cover installing both versions of wal-g 2 and 3 2 as default * chore: add the "v" as requested * major version on pkg name to avoid collision can alias on install * chore: cover in package build as well * chore: better support for both versions, symlink default to v3 on new image * chore: cleanup file * chore: bump version for release * chore: update dockerfiles * chore: get aliases right * chore: remove rendundant * chore: testing suffix * chore: bump for release
1 parent 01978ef commit d1cb4ca

File tree

7 files changed

+134
-146
lines changed

7 files changed

+134
-146
lines changed

‎Dockerfile-15

+12-8
Original file line numberDiff line numberDiff line change
@@ -120,14 +120,14 @@ ENV PGDATA=/var/lib/postgresql/data
120120
####################
121121
FROM base as walg
122122
ARG wal_g_release
123-
# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz
124-
RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \
125-
apt-get update && apt-get install -y --no-install-recommends curl && \
126-
curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \
127-
tar -xvf /tmp/wal-g.tar.gz -C /tmp && \
128-
rm -rf /tmp/wal-g.tar.gz && \
129-
mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g
123+
WORKDIR /nixpg
124+
125+
RUN nix profile install .#wal-g-3 && \
126+
ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g
130127

128+
RUN nix store gc
129+
130+
WORKDIR /
131131
# ####################
132132
# # Download gosu for easy step-down from root
133133
# ####################
@@ -179,7 +179,8 @@ RUN sed -i \
179179
-e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
180180
echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
181181
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
182-
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
182+
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
183+
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
183184
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
184185
usermod -aG postgres wal-g && \
185186
mkdir -p /etc/postgresql-custom && \
@@ -219,4 +220,7 @@ ENV LC_ALL en_US.UTF-8
219220
ENV LC_CTYPE=C.UTF-8
220221
ENV LC_COLLATE=C.UTF-8
221222
ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive
223+
RUN mkdir -p /usr/share/postgresql/extension/ && \
224+
ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \
225+
chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh
222226
CMD ["postgres", "-D", "/etc/postgresql"]

‎Dockerfile-orioledb-17

+14-8
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ ARG pgvector_release=0.4.0
3737
ARG pg_tle_release=1.3.2
3838
ARG index_advisor_release=0.2.0
3939
ARG supautils_release=2.2.0
40-
ARG wal_g_release=2.0.1
40+
ARG wal_g_release=3.0.5
4141

4242
FROM ubuntu:focal as base
4343

@@ -118,14 +118,15 @@ ENV PGDATA=/var/lib/postgresql/data
118118
####################
119119
FROM base as walg
120120
ARG wal_g_release
121-
# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz
122-
RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \
123-
apt-get update && apt-get install -y --no-install-recommends curl && \
124-
curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \
125-
tar -xvf /tmp/wal-g.tar.gz -C /tmp && \
126-
rm -rf /tmp/wal-g.tar.gz && \
127-
mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g
128121

122+
WORKDIR /nixpg
123+
124+
RUN nix profile install .#wal-g-3 && \
125+
ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g
126+
127+
RUN nix store gc
128+
129+
WORKDIR /
129130
# ####################
130131
# # Download gosu for easy step-down from root
131132
# ####################
@@ -178,6 +179,7 @@ RUN sed -i \
178179
echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
179180
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
180181
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
182+
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
181183
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
182184
usermod -aG postgres wal-g && \
183185
mkdir -p /etc/postgresql-custom && \
@@ -231,4 +233,8 @@ ENV LC_ALL en_US.UTF-8
231233
ENV LC_CTYPE=C.UTF-8
232234
ENV LC_COLLATE=C.UTF-8
233235
ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive
236+
RUN mkdir -p /usr/share/postgresql/extension/ && \
237+
ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \
238+
chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh
239+
234240
CMD ["postgres", "-D", "/etc/postgresql"]

‎ansible/tasks/setup-wal-g.yml

+21-70
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,3 @@
1-
# Downloading dependencies
2-
# - name: wal-g dependencies
3-
# become: yes
4-
# apt:
5-
# pkg:
6-
# - libbrotli-dev
7-
# - liblzo2-dev
8-
# - libsodium-dev
9-
# - cmake
10-
11-
# install go dependency for WAL-G
12-
# - name: wal-g go dependency
13-
# get_url:
14-
# url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
15-
# dest: /tmp
16-
# checksum: "{{ golang_version_checksum[platform] }}"
17-
# timeout: 60
18-
19-
# - name: unpack go archive
20-
# unarchive:
21-
# remote_src: yes
22-
# src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
23-
# dest: /usr/local
24-
25-
# # Download WAL-G
26-
# - name: wal-g - download latest version
27-
# git:
28-
# repo: https://github.com/wal-g/wal-g.git
29-
# dest: /tmp/wal-g
30-
# version: "v{{ wal_g_release }}"
31-
# become: yes
32-
33-
# - name: wal-g - pg_clean
34-
# make:
35-
# chdir: /tmp/wal-g
36-
# target: pg_clean
37-
# params:
38-
# GOBIN: "/usr/local/bin"
39-
# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
40-
# USE_LIBSODIUM: true
41-
# become: yes
42-
# ignore_errors: yes
43-
44-
# - name: wal-g - deps
45-
# make:
46-
# chdir: /tmp/wal-g
47-
# target: deps
48-
# params:
49-
# GOBIN: "/usr/local/bin"
50-
# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
51-
# USE_LIBSODIUM: true
52-
# become: yes
53-
# ignore_errors: yes
54-
55-
# - name: wal-g - build and install
56-
# community.general.make:
57-
# chdir: /tmp/wal-g
58-
# target: pg_install
59-
# jobs: "{{ parallel_jobs | default(omit) }}"
60-
# params:
61-
# GOBIN: "/usr/local/bin"
62-
# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
63-
# USE_LIBSODIUM: true
64-
# become: yes
65-
661
- name: Create wal-g group
672
group:
683
name: wal-g
@@ -86,20 +21,36 @@
8621
mode: '0770'
8722
when: nixpkg_mode
8823

89-
- name: Install wal-g from nix binary cache
24+
- name: Install wal-g 2 from nix binary cache
9025
become: yes
9126
shell: |
92-
sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g"
27+
sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-2"
9328
when: stage2_nix
9429

95-
- name: Create symlink for wal-g from Nix profile to /usr/local/bin
30+
- name: Install wal-g 3 from nix binary cache
31+
become: yes
32+
shell: |
33+
sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-3"
34+
when: stage2_nix
35+
36+
- name: Create symlink for wal-g-3 from Nix profile to /usr/local/bin
9637
ansible.builtin.file:
97-
src: /home/wal-g/.nix-profile/bin/wal-g
98-
dest: /usr/local/bin/wal-g
38+
src: /home/wal-g/.nix-profile/bin/wal-g-3
39+
dest: /usr/local/bin/wal-g-v3
9940
state: link
10041
force: yes # This will replace existing file/symlink if it exists
10142
become: yes # Need sudo to write to /usr/local/bin
10243
when: stage2_nix
44+
45+
- name: Create symlink to make wal-g-v2 the default wal-g
46+
ansible.builtin.file:
47+
src: /usr/local/bin/wal-g-v2
48+
dest: /usr/local/bin/wal-g
49+
state: link
50+
force: yes
51+
become: yes
52+
when: stage2_nix
53+
10354
- name: Create /etc/wal-g/config.json
10455
file:
10556
path: /etc/wal-g/config.json

‎ansible/vars.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ postgres_major:
88

99
# Full version strings for each major version
1010
postgres_release:
11-
postgresorioledb-17: "17.0.1.050-orioledb"
12-
postgres15: "15.8.1.057"
11+
postgresorioledb-17: "17.0.1.051-orioledb"
12+
postgres15: "15.8.1.058"
1313

1414
# Non Postgres Extensions
1515
pgbouncer_release: "1.19.0"

‎docker/nix/build_nix.sh

+6-5
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@ SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"')
1010

1111
nix build .#checks.$SYSTEM.psql_15 -L --no-link
1212
nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link
13-
nix build .#psql_15/bin -o psql_15
13+
nix build .#psql_15/bin -o psql_15 -L
1414

15-
nix build .#psql_orioledb-17/bin -o psql_orioledb_17
16-
nix build .#wal-g -o wal-g
15+
nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L
16+
nix build .#wal-g-2 -o wal-g-2 -L
17+
nix build .#wal-g-3 -o wal-g-3 -L
1718
# Copy to S3
18-
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
19+
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-2
20+
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3
1921
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
2022
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17
2123
if [ "$SYSTEM" = "aarch64-linux" ]; then
@@ -27,5 +29,4 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then
2729
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src
2830
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug
2931
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_src
30-
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
3132
fi

‎flake.nix

+3-2
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@
8787
sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
8888
supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
8989
mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
90-
wal-g = pkgs.callPackage ./nix/wal-g.nix { };
90+
inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g-2 wal-g-3;
9191
# Our list of PostgreSQL extensions which come from upstream Nixpkgs.
9292
# These are maintained upstream and can easily be used here just by
9393
# listing their name. Anytime the version of nixpkgs is upgraded, these
@@ -406,7 +406,8 @@
406406
# PostgreSQL versions.
407407
psql_15 = postgresVersions.psql_15;
408408
psql_orioledb-17 = postgresVersions.psql_orioledb-17;
409-
wal-g = wal-g;
409+
wal-g-2 = wal-g-2;
410+
wal-g-3 = wal-g-3;
410411
sfcgal = sfcgal;
411412
pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
412413
inherit postgresql_15 postgresql_orioledb-17;

‎nix/wal-g.nix

+76-51
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,83 @@
1-
{
2-
lib,
3-
buildGoModule,
4-
fetchFromGitHub,
5-
brotli,
6-
libsodium,
7-
installShellFiles,
1+
{ lib
2+
, buildGoModule
3+
, fetchFromGitHub
4+
, brotli
5+
, libsodium
6+
, installShellFiles
7+
,
88
}:
99

10-
buildGoModule rec {
11-
pname = "wal-g";
12-
version = "3.0.5";
10+
let
11+
walGCommon = { version, vendorHash, sha256, majorVersion }:
12+
buildGoModule rec {
13+
pname = "wal-g-${majorVersion}";
14+
inherit version;
1315

14-
src = fetchFromGitHub {
15-
owner = "wal-g";
16-
repo = "wal-g";
17-
rev = "v${version}";
18-
sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
16+
src = fetchFromGitHub {
17+
owner = "wal-g";
18+
repo = "wal-g";
19+
rev = "v${version}";
20+
inherit sha256;
21+
};
22+
23+
inherit vendorHash;
24+
25+
nativeBuildInputs = [ installShellFiles ];
26+
27+
buildInputs = [
28+
brotli
29+
libsodium
30+
];
31+
32+
subPackages = [ "main/pg" ];
33+
34+
tags = [
35+
"brotli"
36+
"libsodium"
37+
];
38+
39+
ldflags = [
40+
"-s"
41+
"-w"
42+
"-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
43+
"-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
44+
];
45+
46+
postInstall = ''
47+
mv $out/bin/pg $out/bin/wal-g-${majorVersion}
48+
49+
# Create version-specific completions
50+
mkdir -p $out/share/bash-completion/completions
51+
$out/bin/wal-g-${majorVersion} completion bash > $out/share/bash-completion/completions/wal-g-${majorVersion}
52+
53+
mkdir -p $out/share/zsh/site-functions
54+
$out/bin/wal-g-${majorVersion} completion zsh > $out/share/zsh/site-functions/_wal-g-${majorVersion}
55+
56+
'';
57+
58+
meta = with lib; {
59+
homepage = "https://github.com/wal-g/wal-g";
60+
license = licenses.asl20;
61+
description = "Archival restoration tool for PostgreSQL";
62+
mainProgram = "wal-g-${majorVersion}";
63+
maintainers = [ samrose ];
64+
};
65+
};
66+
in
67+
{
68+
# wal-g v2.0.1
69+
wal-g-2 = walGCommon {
70+
version = "2.0.1";
71+
sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok=";
72+
vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g=";
73+
majorVersion = "2";
1974
};
2075

21-
vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
22-
23-
nativeBuildInputs = [ installShellFiles ];
24-
25-
buildInputs = [
26-
brotli
27-
libsodium
28-
];
29-
30-
subPackages = [ "main/pg" ];
31-
32-
tags = [
33-
"brotli"
34-
"libsodium"
35-
];
36-
37-
ldflags = [
38-
"-s"
39-
"-w"
40-
"-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
41-
"-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
42-
];
43-
44-
postInstall = ''
45-
mv $out/bin/pg $out/bin/wal-g
46-
installShellCompletion --cmd wal-g \
47-
--bash <($out/bin/wal-g completion bash) \
48-
--zsh <($out/bin/wal-g completion zsh)
49-
'';
50-
51-
meta = with lib; {
52-
homepage = "https://github.com/wal-g/wal-g";
53-
license = licenses.asl20;
54-
description = "Archival restoration tool for PostgreSQL";
55-
mainProgram = "wal-g";
56-
maintainers = [ samrose ];
76+
# wal-g v3.0.5
77+
wal-g-3 = walGCommon {
78+
version = "3.0.5";
79+
sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
80+
vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
81+
majorVersion = "3";
5782
};
5883
}

0 commit comments

Comments
 (0)