chore: mv ansible migration test to nix flake check

migration tests are now run on `nix flake check`:

- tests for some extensions are adjusted to run conditionally on non-orioledb
  + preserves behavior from the ansible tests
- ebssurrogate/files/unit-tests/unit-test-01.sql are deleted
  + these tests are already included nix/test/prime.sql

cleanup ansible/tasks/test-image.yml:

- remove pg_prove since it's no longer used
- remove unit tests since the same logic is already included
  nix/tests/prime.sql
- remove migration tests since it's now tested with nix flake check

Author: steve-chavez

Diff for: amazon-arm64-nix.pkr.hcl

@@ -228,11 +228,6 @@ build {
     destination = "/tmp"
   }
 
-  provisioner "file" {
-    source = "ebssurrogate/files/unit-tests"
-    destination = "/tmp"
-  }
-
   # Copy ansible playbook
   provisioner "shell" {
     inline = ["mkdir /tmp/ansible-playbook"]

Diff for: ansible/tasks/test-image.yml

@@ -1,9 +1,3 @@
-- name: install pg_prove
-  apt:
-    pkg:
-      - libtap-parser-sourcehandler-pgtap-perl
-  when: debpkg_mode
-
 # - name: Temporarily disable PG Sodium references in config
 #   become: yes
 #   become_user: postgres
@@ -16,9 +10,9 @@
   become_user: postgres
   shell:
     cmd: >
-      sed -i.bak 
-      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
-      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+      sed -i.bak
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/'
       -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
       -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
       /etc/postgresql/postgresql.conf
@@ -74,53 +68,6 @@
     LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
   when: stage2_nix
 
-
-- name: Check psql_version and modify migrations if oriole-xx
-  block:
-    - name: Check if psql_version is psql_orioledb-xx
-      set_fact:
-        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
-
-    - name: Remove specified extensions from SQL file
-      ansible.builtin.command:
-        cmd: >
-          sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql
-      when: is_psql_oriole
-      become: yes
-
-    - name: Remove specified extension files from extensions directory
-      ansible.builtin.find:
-        paths: /tmp/migrations/tests/extensions
-        patterns: 
-          - '*timescaledb*.sql'
-          - '*plv8*.sql'
-          - '*postgis*.sql'
-          - '*pgrouting*.sql'
-      register: files_to_remove
-      when: is_psql_oriole
-
-    - name: Delete matched extension files
-      ansible.builtin.file:
-        path: "{{ item.path }}"
-        state: absent
-      loop: "{{ files_to_remove.files }}"
-      when: is_psql_oriole
-      become: yes
-
-- name: Run Unit tests (with filename unit-test-*) on Postgres Database
-  shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql
-  register: retval
-  failed_when: retval.rc != 0
-  when: debpkg_mode or stage2_nix
-
-- name: Run migrations tests
-  shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql
-  register: retval
-  failed_when: retval.rc != 0
-  when: debpkg_mode or stage2_nix
-  args:
-    chdir: /tmp/migrations
-
 - name: Re-enable PG Sodium references in config
   become: yes
   become_user: postgres
@@ -132,14 +79,6 @@
   shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();'
   when: debpkg_mode or stage2_nix
 
-- name: remove pg_prove
-  apt:
-    pkg:
-      - libtap-parser-sourcehandler-pgtap-perl
-    state: absent
-    autoremove: yes
-  when: debpkg_mode
-
 - name: Stop Postgres Database
   become: yes
   become_user: postgres

Diff for: ebssurrogate/files/unit-tests/unit-test-01.sql

@@ -180,9 +180,6 @@ function setup_chroot_environment {
 	# Copy migrations
 	cp -r /tmp/migrations /mnt/tmp/
 
-	# Copy unit tests 
-	cp -r /tmp/unit-tests /mnt/tmp/
-
 	# Copy the bootstrap script into place and execute inside chroot
 	cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh
 	chroot /mnt /tmp/chroot-bootstrap-nix.sh

Diff for: ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -807,6 +807,9 @@
                 exit 1
               fi
 
+              echo "Running migrations tests"
+              pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql
+
               mkdir -p $out/regression_output
               if ! pg_regress \
                 --use-existing \

Diff for: flake.nix

@@ -1,43 +1,38 @@
-BEGIN;
-create extension if not exists postgis_sfcgal with schema "extensions" cascade;
-ROLLBACK;
+begin;
+do $_$
+begin
+  if not exists (select 1 from pg_extension where extname = 'orioledb') then
+    -- create postgis tiger as supabase_admin
+    create extension if not exists postgis_tiger_geocoder cascade;
 
-BEGIN;
-create extension if not exists postgis_raster with schema "extensions" cascade;
-ROLLBACK;
+    -- \ir ansible/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql
+    grant usage on schema tiger, tiger_data to postgres with grant option;
+    grant all privileges on all tables in schema tiger, tiger_data to postgres with grant option;
+    grant all privileges on all routines in schema tiger, tiger_data to postgres with grant option;
+    grant all privileges on all sequences in schema tiger, tiger_data to postgres with grant option;
+    alter default privileges in schema tiger, tiger_data grant all on tables to postgres with grant option;
+    alter default privileges in schema tiger, tiger_data grant all on routines to postgres with grant option;
+    alter default privileges in schema tiger, tiger_data grant all on sequences to postgres with grant option;
+    SET search_path TO extensions, public, tiger, tiger_data;
+    -- postgres role should have access
+    set local role postgres;
+    perform tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
 
-BEGIN;
--- create postgis tiger as supabase_admin
-create extension if not exists address_standardizer with schema extensions;
-create extension if not exists postgis_tiger_geocoder cascade;
+    -- other roles can be granted access
+    grant usage on schema tiger, tiger_data to authenticated;
+    grant select on all tables in schema tiger, tiger_data to authenticated;
+    grant execute on all routines in schema tiger, tiger_data to authenticated;
 
--- \ir ansible/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql
-grant usage on schema tiger, tiger_data to postgres with grant option;
-grant all privileges on all tables in schema tiger, tiger_data to postgres with grant option;
-grant all privileges on all routines in schema tiger, tiger_data to postgres with grant option;
-grant all privileges on all sequences in schema tiger, tiger_data to postgres with grant option;
-alter default privileges in schema tiger, tiger_data grant all on tables to postgres with grant option;
-alter default privileges in schema tiger, tiger_data grant all on routines to postgres with grant option;
-alter default privileges in schema tiger, tiger_data grant all on sequences to postgres with grant option;
-SET search_path TO extensions, public, tiger, tiger_data;
--- postgres role should have access
-set local role postgres;
-select tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
+    -- authenticated role should have access now
+    set local role authenticated;
+    perform tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
+    reset role;
 
--- other roles can be granted access
-grant usage on schema tiger, tiger_data to authenticated;
-grant select on all tables in schema tiger, tiger_data to authenticated;
-grant execute on all routines in schema tiger, tiger_data to authenticated;
-
--- authenticated role should have access now
-set local role authenticated;
-select tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
-ROLLBACK;
-
-BEGIN;
--- address standardizer creates a table in extensions schema, owned by supabase_admin
-create extension if not exists address_standardizer_data_us with schema extensions;
--- postgres role should have access
-set local role postgres;
-select * from extensions.us_lex;
-ROLLBACK;
+    -- postgres role should have access to address_standardizer_data_us
+    set local role postgres;
+    perform 1 from us_lex;
+    reset role;
+  end if;
+end
+$_$;
+rollback;

Diff for: migrations/tests/extensions/01-postgis.sql

@@ -1,3 +1,9 @@
-BEGIN;
-create extension if not exists pgrouting with schema "extensions" cascade;
-ROLLBACK;
+begin;
+do $_$
+begin
+  if not exists (select 1 from pg_extension where extname = 'orioledb') then
+    create extension if not exists pgrouting with schema "extensions" cascade;
+  end if;
+end
+$_$;
+rollback;

Diff for: migrations/tests/extensions/02-pgrouting.sql

@@ -1,3 +1,9 @@
-BEGIN;
-create extension if not exists timescaledb with schema "extensions";
-ROLLBACK;
+begin;
+do $_$
+begin
+  if not exists (select 1 from pg_extension where extname = 'orioledb') then
+    create extension if not exists timescaledb with schema "extensions";
+  end if;
+end
+$_$;
+rollback;

Diff for: migrations/tests/extensions/10-timescaledb.sql

@@ -1,3 +1,9 @@
-BEGIN;
-create extension if not exists plv8 with schema "pg_catalog";
-ROLLBACK;
+begin;
+do $_$
+begin
+  if not exists (select 1 from pg_extension where extname = 'orioledb') then
+    create extension if not exists plv8 with schema "pg_catalog";
+  end if;
+end
+$_$;
+rollback;

Diff for: migrations/tests/extensions/13-plv8.sql

@@ -122,11 +122,6 @@ build {
     destination = "/tmp"
   }
 
-  provisioner "file" {
-    source = "ebssurrogate/files/unit-tests"
-    destination = "/tmp"
-  }
-
   provisioner "shell" {
     environment_vars = [
       "POSTGRES_MAJOR_VERSION=${var.postgres-major-version}",

Diff for: qemu-arm64-nix.pkr.hcl

@@ -73,17 +73,17 @@ source "amazon-ebs" "ubuntu" {
     most_recent = true
     owners      = ["amazon", "self"]
   }
-  
+
   communicator = "ssh"
   ssh_pty = true
   ssh_username = "ubuntu"
   ssh_timeout = "5m"
-  
+
   associate_public_ip_address = true
 
 
   ena_support = true
-  
+
   run_tags = {
     creator           = "packer"
     appType           = "postgres"
@@ -126,22 +126,17 @@ build {
     destination = "/tmp"
   }
 
-  provisioner "file" {
-    source       = "ebssurrogate/files/unit-tests"
-    destination  = "/tmp/unit-tests"
-  }
-
   provisioner "file" {
     source = "scripts"
     destination = "/tmp/ansible-playbook"
   }
-  
+
   provisioner "shell" {
     environment_vars = [
       "GIT_SHA=${var.git_sha}",
       "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}"
     ]
      script = "scripts/nix-provision.sh"
   }
-  
+
 }