From ef23d0dd1c2d9a93d0a6044214ea36d97880e994 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Feb 2025 12:22:19 -0500 Subject: [PATCH 01/17] feature: wal-g v 3 and v 2 --- nix/wal-g.nix | 108 +++++++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/nix/wal-g.nix b/nix/wal-g.nix index 13f2bbc90..1db3a3e79 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -7,52 +7,70 @@ installShellFiles, }: -buildGoModule rec { - pname = "wal-g"; - version = "3.0.5"; - - src = fetchFromGitHub { - owner = "wal-g"; - repo = "wal-g"; - rev = "v${version}"; - sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U="; +let + walGCommon = { version, vendorHash, sha256 }: + buildGoModule rec { + pname = "wal-g"; + inherit version; + + src = fetchFromGitHub { + owner = "wal-g"; + repo = "wal-g"; + rev = "v${version}"; + inherit sha256; + }; + + inherit vendorHash; + + nativeBuildInputs = [ installShellFiles ]; + + buildInputs = [ + brotli + libsodium + ]; + + subPackages = [ "main/pg" ]; + + tags = [ + "brotli" + "libsodium" + ]; + + ldflags = [ + "-s" + "-w" + "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}" + "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}" + ]; + + postInstall = '' + mv $out/bin/pg $out/bin/wal-g + installShellCompletion --cmd wal-g \ + --bash <($out/bin/wal-g completion bash) \ + --zsh <($out/bin/wal-g completion zsh) + ''; + + meta = with lib; { + homepage = "https://github.com/wal-g/wal-g"; + license = licenses.asl20; + description = "Archival restoration tool for PostgreSQL"; + mainProgram = "wal-g"; + maintainers = [ samrose ]; + }; + }; +in +{ + # wal-g v2.0.1 + wal-g = walGCommon { + version = "2.0.1"; + sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; # Replace with actual sha256 for v2.0.1 + vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g="; # Replace with actual vendorHash for v2.0.1 }; - vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; - - nativeBuildInputs = [ installShellFiles ]; - - buildInputs = [ - brotli - libsodium - ]; - - subPackages = [ "main/pg" ]; - - tags = [ - "brotli" - "libsodium" - ]; - - ldflags = [ - "-s" - "-w" - "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}" - "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}" - ]; - - postInstall = '' - mv $out/bin/pg $out/bin/wal-g - installShellCompletion --cmd wal-g \ - --bash <($out/bin/wal-g completion bash) \ - --zsh <($out/bin/wal-g completion zsh) - ''; - - meta = with lib; { - homepage = "https://github.com/wal-g/wal-g"; - license = licenses.asl20; - description = "Archival restoration tool for PostgreSQL"; - mainProgram = "wal-g"; - maintainers = [ samrose ]; + # wal-g v3.0.5 + wal-g-3 = walGCommon { + version = "3.0.5"; + sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U="; + vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; }; } From 5ecebe77890ae94817421aca9fe10241ea988e28 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Feb 2025 13:05:50 -0500 Subject: [PATCH 02/17] chore: install, symlink --- ansible/tasks/setup-wal-g.yml | 83 +++++++---------------------------- 1 file changed, 17 insertions(+), 66 deletions(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 9907d3ee7..e3a612634 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -1,68 +1,3 @@ -# Downloading dependencies -# - name: wal-g dependencies -# become: yes -# apt: -# pkg: -# - libbrotli-dev -# - liblzo2-dev -# - libsodium-dev -# - cmake - -# install go dependency for WAL-G -# - name: wal-g go dependency -# get_url: -# url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" -# dest: /tmp -# checksum: "{{ golang_version_checksum[platform] }}" -# timeout: 60 - -# - name: unpack go archive -# unarchive: -# remote_src: yes -# src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" -# dest: /usr/local - -# # Download WAL-G -# - name: wal-g - download latest version -# git: -# repo: https://github.com/wal-g/wal-g.git -# dest: /tmp/wal-g -# version: "v{{ wal_g_release }}" -# become: yes - -# - name: wal-g - pg_clean -# make: -# chdir: /tmp/wal-g -# target: pg_clean -# params: -# GOBIN: "/usr/local/bin" -# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# USE_LIBSODIUM: true -# become: yes -# ignore_errors: yes - -# - name: wal-g - deps -# make: -# chdir: /tmp/wal-g -# target: deps -# params: -# GOBIN: "/usr/local/bin" -# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# USE_LIBSODIUM: true -# become: yes -# ignore_errors: yes - -# - name: wal-g - build and install -# community.general.make: -# chdir: /tmp/wal-g -# target: pg_install -# jobs: "{{ parallel_jobs | default(omit) }}" -# params: -# GOBIN: "/usr/local/bin" -# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# USE_LIBSODIUM: true -# become: yes - - name: Create wal-g group group: name: wal-g @@ -86,12 +21,18 @@ mode: '0770' when: nixpkg_mode -- name: Install wal-g from nix binary cache +- name: Install wal-g 2 from nix binary cache become: yes shell: | sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g" when: stage2_nix +- name: Install wal-g 3 from nix binary cache + become: yes + shell: | + sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-3" + when: stage2_nix + - name: Create symlink for wal-g from Nix profile to /usr/local/bin ansible.builtin.file: src: /home/wal-g/.nix-profile/bin/wal-g @@ -100,6 +41,16 @@ force: yes # This will replace existing file/symlink if it exists become: yes # Need sudo to write to /usr/local/bin when: stage2_nix + +- name: Create symlink for wal-g-3 from Nix profile to /usr/local/bin + ansible.builtin.file: + src: /home/wal-g/.nix-profile/bin/wal-g-3 + dest: /usr/local/bin/wal-g-3 + state: link + force: yes # This will replace existing file/symlink if it exists + become: yes # Need sudo to write to /usr/local/bin + when: stage2_nix + - name: Create /etc/wal-g/config.json file: path: /etc/wal-g/config.json From b49b77ab221de939f2fe2e8f72f384be67413d82 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 28 Feb 2025 14:26:53 -0500 Subject: [PATCH 03/17] chore: bump versions to test wal-g --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 88fe86749..784b15dcc 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.050-orioledb" - postgres15: "15.8.1.057" + postgresorioledb-17: "17.0.1.041-orioledb-walg-1" + postgres15: "15.8.1.047-walg-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 6306e28b135d175167c00d2a55984aefeb02e311 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 5 Mar 2025 11:38:08 -0500 Subject: [PATCH 04/17] chore: build wal-g in ci and copy to binary cache on success --- nix/wal-g.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/wal-g.nix b/nix/wal-g.nix index 1db3a3e79..7712f9e5a 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -63,8 +63,8 @@ in # wal-g v2.0.1 wal-g = walGCommon { version = "2.0.1"; - sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; # Replace with actual sha256 for v2.0.1 - vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g="; # Replace with actual vendorHash for v2.0.1 + sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; + vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g="; }; # wal-g v3.0.5 From 2d9e7299c072bc4322ff3d2825c306112201f33c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 18 Mar 2025 16:45:11 -0400 Subject: [PATCH 05/17] chore: release wal-g 3 --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 784b15dcc..d8e963b70 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.041-orioledb-walg-1" - postgres15: "15.8.1.047-walg-1" + postgresorioledb-17: "17.0.1.049-orioledb" + postgres15: "15.8.1.056" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 2631d148488e7e559f14c6e94ff9959a1c7540ad Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 08:36:41 -0400 Subject: [PATCH 06/17] chore: cover installing both versions of wal-g 2 and 3 2 as default --- ansible/vars.yml | 4 ++-- docker/nix/build_nix.sh | 9 +++++---- flake.nix | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d8e963b70..6c56ffe81 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.049-orioledb" - postgres15: "15.8.1.056" + postgresorioledb-17: "17.0.1.050-orioledb-walg" + postgres15: "15.8.1.057-walg" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 48286907e..d88cd87e7 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,12 +10,14 @@ SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') nix build .#checks.$SYSTEM.psql_15 -L --no-link nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link -nix build .#psql_15/bin -o psql_15 +nix build .#psql_15/bin -o psql_15 -L -nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -nix build .#wal-g -o wal-g +nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L +nix build .#wal-g -o wal-g -L +nix build .#wal-g-3 -o wal-g-3 -L # Copy to S3 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 if [ "$SYSTEM" = "aarch64-linux" ]; then @@ -27,5 +29,4 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_src - nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g fi diff --git a/flake.nix b/flake.nix index dadc55253..dd1dcf6e3 100644 --- a/flake.nix +++ b/flake.nix @@ -87,7 +87,7 @@ sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { }; - wal-g = pkgs.callPackage ./nix/wal-g.nix { }; + inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g wal-g-3; # Our list of PostgreSQL extensions which come from upstream Nixpkgs. # These are maintained upstream and can easily be used here just by # listing their name. Anytime the version of nixpkgs is upgraded, these @@ -407,6 +407,7 @@ psql_15 = postgresVersions.psql_15; psql_orioledb-17 = postgresVersions.psql_orioledb-17; wal-g = wal-g; + wal-g-3 = wal-g-3; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_orioledb-17; From b63b574cd3af6f37cea61fe9f2cca9979c60a835 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 08:46:13 -0400 Subject: [PATCH 07/17] chore: add the "v" as requested --- ansible/tasks/setup-wal-g.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index e3a612634..1eb91adc7 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -45,7 +45,7 @@ - name: Create symlink for wal-g-3 from Nix profile to /usr/local/bin ansible.builtin.file: src: /home/wal-g/.nix-profile/bin/wal-g-3 - dest: /usr/local/bin/wal-g-3 + dest: /usr/local/bin/wal-g-v3 state: link force: yes # This will replace existing file/symlink if it exists become: yes # Need sudo to write to /usr/local/bin From 3bdfc6607c6539b82f9c162aa1cd36fcf93d578a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 12:24:47 -0400 Subject: [PATCH 08/17] major version on pkg name to avoid collision can alias on install --- ansible/tasks/setup-wal-g.yml | 4 ++-- flake.nix | 4 ++-- nix/wal-g.nix | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 1eb91adc7..c7bc8cf29 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -24,7 +24,7 @@ - name: Install wal-g 2 from nix binary cache become: yes shell: | - sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g" + sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-2" when: stage2_nix - name: Install wal-g 3 from nix binary cache @@ -35,7 +35,7 @@ - name: Create symlink for wal-g from Nix profile to /usr/local/bin ansible.builtin.file: - src: /home/wal-g/.nix-profile/bin/wal-g + src: /home/wal-g/.nix-profile/bin/wal-g-2 dest: /usr/local/bin/wal-g state: link force: yes # This will replace existing file/symlink if it exists diff --git a/flake.nix b/flake.nix index dd1dcf6e3..b36ccda57 100644 --- a/flake.nix +++ b/flake.nix @@ -87,7 +87,7 @@ sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { }; - inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g wal-g-3; + inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g-2 wal-g-3; # Our list of PostgreSQL extensions which come from upstream Nixpkgs. # These are maintained upstream and can easily be used here just by # listing their name. Anytime the version of nixpkgs is upgraded, these @@ -406,7 +406,7 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_orioledb-17 = postgresVersions.psql_orioledb-17; - wal-g = wal-g; + wal-g-2 = wal-g-2; wal-g-3 = wal-g-3; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; diff --git a/nix/wal-g.nix b/nix/wal-g.nix index 7712f9e5a..087ace00d 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -8,9 +8,9 @@ }: let - walGCommon = { version, vendorHash, sha256 }: + walGCommon = { version, vendorHash, sha256, majorVersion }: buildGoModule rec { - pname = "wal-g"; + pname = "wal-g-${majorVersion}"; inherit version; src = fetchFromGitHub { @@ -61,10 +61,11 @@ let in { # wal-g v2.0.1 - wal-g = walGCommon { + wal-g-2 = walGCommon { version = "2.0.1"; sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g="; + majorVersion = "2"; }; # wal-g v3.0.5 @@ -72,5 +73,6 @@ in version = "3.0.5"; sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U="; vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; + majorVersion = "3"; }; } From 899c23570a162364b3928cbbe73500fab346709c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 12:30:32 -0400 Subject: [PATCH 09/17] chore: cover in package build as well --- docker/nix/build_nix.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index d88cd87e7..135031afc 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -13,10 +13,10 @@ nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link nix build .#psql_15/bin -o psql_15 -L nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L -nix build .#wal-g -o wal-g -L +nix build .#wal-g-2 -o wal-g-2 -L nix build .#wal-g-3 -o wal-g-3 -L # Copy to S3 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-2 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 From 0b6cdd74289258eac8d106481b83e4b6ef025571 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 13:31:36 -0400 Subject: [PATCH 10/17] chore: better support for both versions, symlink default to v3 on new image --- ansible/tasks/setup-wal-g.yml | 11 ++++++++++- nix/wal-g.nix | 17 +++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index c7bc8cf29..c20a64c62 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -36,7 +36,7 @@ - name: Create symlink for wal-g from Nix profile to /usr/local/bin ansible.builtin.file: src: /home/wal-g/.nix-profile/bin/wal-g-2 - dest: /usr/local/bin/wal-g + dest: /usr/local/bin/wal-g-v2 state: link force: yes # This will replace existing file/symlink if it exists become: yes # Need sudo to write to /usr/local/bin @@ -51,6 +51,15 @@ become: yes # Need sudo to write to /usr/local/bin when: stage2_nix +- name: Create symlink to make wal-g-v3 the default wal-g + ansible.builtin.file: + src: /usr/local/bin/wal-g-v3 + dest: /usr/local/bin/wal-g + state: link + force: yes + become: yes + when: stage2_nix + - name: Create /etc/wal-g/config.json file: path: /etc/wal-g/config.json diff --git a/nix/wal-g.nix b/nix/wal-g.nix index 087ace00d..e58342da8 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -44,17 +44,22 @@ let ]; postInstall = '' - mv $out/bin/pg $out/bin/wal-g - installShellCompletion --cmd wal-g \ - --bash <($out/bin/wal-g completion bash) \ - --zsh <($out/bin/wal-g completion zsh) + mv $out/bin/pg $out/bin/wal-g-${majorVersion} + + # Create version-specific completions + mkdir -p $out/share/bash-completion/completions + $out/bin/wal-g-${majorVersion} completion bash > $out/share/bash-completion/completions/wal-g-${majorVersion} + + mkdir -p $out/share/zsh/site-functions + $out/bin/wal-g-${majorVersion} completion zsh > $out/share/zsh/site-functions/_wal-g-${majorVersion} + ''; meta = with lib; { homepage = "https://github.com/wal-g/wal-g"; license = licenses.asl20; description = "Archival restoration tool for PostgreSQL"; - mainProgram = "wal-g"; + mainProgram = "wal-g-${majorVersion}"; maintainers = [ samrose ]; }; }; @@ -75,4 +80,4 @@ in vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; majorVersion = "3"; }; -} +} \ No newline at end of file From c36c1827d532c525f3b4fc34236e986411a895a0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 13:33:06 -0400 Subject: [PATCH 11/17] chore: cleanup file --- nix/wal-g.nix | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nix/wal-g.nix b/nix/wal-g.nix index e58342da8..764b1c9e4 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -1,10 +1,10 @@ -{ - lib, - buildGoModule, - fetchFromGitHub, - brotli, - libsodium, - installShellFiles, +{ lib +, buildGoModule +, fetchFromGitHub +, brotli +, libsodium +, installShellFiles +, }: let @@ -68,7 +68,7 @@ in # wal-g v2.0.1 wal-g-2 = walGCommon { version = "2.0.1"; - sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; + sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g="; majorVersion = "2"; }; @@ -80,4 +80,4 @@ in vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; majorVersion = "3"; }; -} \ No newline at end of file +} From 2761d73e9617aa41f4c8a09d226bbaa183838fe4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 19 Mar 2025 19:32:03 -0400 Subject: [PATCH 12/17] chore: bump version for release --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 6c56ffe81..495b65858 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.050-orioledb-walg" - postgres15: "15.8.1.057-walg" + postgresorioledb-17: "17.0.1.051-orioledb" + postgres15: "15.8.1.058" # Non Postgres Extensions pgbouncer_release: "1.19.0" From ed3a4c317168764a497fb36e6960511bc259e228 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Mar 2025 05:11:43 -0400 Subject: [PATCH 13/17] chore: update dockerfiles --- Dockerfile-15 | 20 ++++++++++++-------- Dockerfile-orioledb-17 | 22 ++++++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Dockerfile-15 b/Dockerfile-15 index 6224767ff..7b1d7099f 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -120,14 +120,14 @@ ENV PGDATA=/var/lib/postgresql/data #################### FROM base as walg ARG wal_g_release -# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz -RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ - apt-get update && apt-get install -y --no-install-recommends curl && \ - curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ - tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ - rm -rf /tmp/wal-g.tar.gz && \ - mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g +WORKDIR /nixpg + +RUN nix profile install .#wal-g-3 && \ + ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g +RUN nix store gc + +WORKDIR / # #################### # # Download gosu for easy step-down from root # #################### @@ -179,7 +179,8 @@ RUN sed -i \ -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ - echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ usermod -aG postgres wal-g && \ mkdir -p /etc/postgresql-custom && \ @@ -219,4 +220,7 @@ ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE=C.UTF-8 ENV LC_COLLATE=C.UTF-8 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +RUN mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index 5f8fed70e..fe9b902cb 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -37,7 +37,7 @@ ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 ARG supautils_release=2.2.0 -ARG wal_g_release=2.0.1 +ARG wal_g_release=3.0.5 FROM ubuntu:focal as base @@ -118,14 +118,15 @@ ENV PGDATA=/var/lib/postgresql/data #################### FROM base as walg ARG wal_g_release -# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz -RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ - apt-get update && apt-get install -y --no-install-recommends curl && \ - curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ - tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ - rm -rf /tmp/wal-g.tar.gz && \ - mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g +WORKDIR /nixpg + +RUN nix profile install .#wal-g-3 && \ + ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g + +RUN nix store gc + +WORKDIR / # #################### # # Download gosu for easy step-down from root # #################### @@ -178,6 +179,7 @@ RUN sed -i \ echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ usermod -aG postgres wal-g && \ mkdir -p /etc/postgresql-custom && \ @@ -231,4 +233,8 @@ ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE=C.UTF-8 ENV LC_COLLATE=C.UTF-8 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +RUN mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + CMD ["postgres", "-D", "/etc/postgresql"] From 93a6cf8891f9098819eec0ff619816c963ad2087 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Mar 2025 11:07:06 -0400 Subject: [PATCH 14/17] chore: get aliases right --- ansible/tasks/setup-wal-g.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index c20a64c62..f6d71c832 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -36,7 +36,7 @@ - name: Create symlink for wal-g from Nix profile to /usr/local/bin ansible.builtin.file: src: /home/wal-g/.nix-profile/bin/wal-g-2 - dest: /usr/local/bin/wal-g-v2 + dest: /usr/local/bin/wal-g state: link force: yes # This will replace existing file/symlink if it exists become: yes # Need sudo to write to /usr/local/bin @@ -53,7 +53,7 @@ - name: Create symlink to make wal-g-v3 the default wal-g ansible.builtin.file: - src: /usr/local/bin/wal-g-v3 + src: /usr/local/bin/wal-g-v2 dest: /usr/local/bin/wal-g state: link force: yes From 6f7b01f219dda3da7ab82baa9c2d24a9e3289a5f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Mar 2025 11:29:59 -0400 Subject: [PATCH 15/17] chore: remove rendundant --- ansible/tasks/setup-wal-g.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index f6d71c832..5ba268cf9 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -33,15 +33,6 @@ sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-3" when: stage2_nix -- name: Create symlink for wal-g from Nix profile to /usr/local/bin - ansible.builtin.file: - src: /home/wal-g/.nix-profile/bin/wal-g-2 - dest: /usr/local/bin/wal-g - state: link - force: yes # This will replace existing file/symlink if it exists - become: yes # Need sudo to write to /usr/local/bin - when: stage2_nix - - name: Create symlink for wal-g-3 from Nix profile to /usr/local/bin ansible.builtin.file: src: /home/wal-g/.nix-profile/bin/wal-g-3 @@ -51,7 +42,7 @@ become: yes # Need sudo to write to /usr/local/bin when: stage2_nix -- name: Create symlink to make wal-g-v3 the default wal-g +- name: Create symlink to make wal-g-v2 the default wal-g ansible.builtin.file: src: /usr/local/bin/wal-g-v2 dest: /usr/local/bin/wal-g From 7ce55bd1501e904599dddd2f00e3a80b7338ceab Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Mar 2025 13:49:00 -0400 Subject: [PATCH 16/17] chore: testing suffix --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 495b65858..1437f1355 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.051-orioledb" - postgres15: "15.8.1.058" + postgresorioledb-17: "17.0.1.051-orioledb-walg" + postgres15: "15.8.1.058-walg" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 73300c5083162b760689b173930aadb16126a431 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Mar 2025 16:38:49 -0400 Subject: [PATCH 17/17] chore: bump for release --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 1437f1355..495b65858 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.051-orioledb-walg" - postgres15: "15.8.1.058-walg" + postgresorioledb-17: "17.0.1.051-orioledb" + postgres15: "15.8.1.058" # Non Postgres Extensions pgbouncer_release: "1.19.0"