add peadm_spec::add_replica test plan for peadm::add_replica

Author: timidri

Diff for: .github/workflows/test-add-replica.yaml

@@ -83,7 +83,7 @@ jobs:
             echo STEP_START=$(date +%s) >> $GITHUB_ENV
           echo ::endgroup::
 
-      - name: 'Provision test cluster'
+      - name: 'Provision test cluster (specified architecture with added DR)'
         timeout-minutes: 15
         run: |
           echo ::group::prepare
@@ -143,7 +143,7 @@ jobs:
         timeout-minutes: 10
         run: |
           buildevents cmd $TRACE_ID $STEP_ID 'bolt plan run peadm_spec::add_replica' -- \
-            bundle exec bolt plan run peadm_spec::add_replica \
+            bundle exec bolt plan run peadm_spec::add_replica -v \
               --inventoryfile spec/fixtures/litmus_inventory.yaml \
               --modulepath spec/fixtures/modules
               

Diff for: plans/add_replica.pp

@@ -1,7 +1,12 @@
-# use case 1: the replica is gone, 
-# use case 2: there is no replica, 
-# both use cases: we have a fresh new VM we want to provision the replica to
-
+# @summary Add a replica to a PE architecture or replace an existing one.
+# Supported use cases:
+#  1: the replica is gone, 
+#  2: there is no replica, 
+#  both use cases: we have a fresh new VM we want to provision the replica to.
+# @param primary_host - The hostname and certname of the primary Puppet server
+# @param replica_host - The hostname and certname of the replica VM
+# @param replica_postgresql_host - The hostname and certname of the host with the replica PE-PosgreSQL database. 
+# Can be a separate host in an XL architecture, or the replica host in standard or Large.
 plan peadm::add_replica(
   Peadm::SingleTargetSpec $primary_host,
   Peadm::SingleTargetSpec $replica_host,
@@ -16,23 +21,34 @@
   $primary_avail_group_letter = $certdata['extensions'][peadm::oid('peadm_availability_group')]
   $replica_avail_group_letter = $primary_avail_group_letter ? { 'A' => 'B', default => 'A' }
 
-  # Stop puppet.service on replica postgresql host
-  run_command('systemctl stop puppet.service', $replica_postgresql_target)
+  # Stop puppet.service on replica postgresql host. Catch errors in case no puppet service is running
+  run_command('systemctl stop puppet.service', $replica_postgresql_target, _catch_errors => true)
 
   # stop pe-postgresql.service on replica postgresql host
-  run_command('systemctl stop pe-postgresql.service', $replica_postgresql_target)
+  run_command('systemctl stop pe-postgresql.service', $replica_postgresql_target, _catch_errors => true)
 
-  run_command("puppet infrastructure forget ${replica_target.peadm::certname()}", $primary_target)
+  run_command("puppet infrastructure forget ${replica_target.peadm::certname()}", $primary_target, _catch_errors => true)
 
   run_task('peadm::agent_install', $replica_target,
     server        => $primary_target.peadm::certname(),
     install_flags => [
-      "extension_requests:${peadm::oid('peadm_role')}='puppet/server'",
+      "extension_requests:${peadm::oid('peadm_role')}=puppet/server",
       "extension_requests:${peadm::oid('peadm_availability_group')}=${replica_avail_group_letter}",
       "main:certname=${replica_target.peadm::certname()}",
     ],
   )
 
+  # run puppet agent once ignoring possible cert errors
+  run_task('peadm::puppet_runonce', $replica_target, _catch_errors => true)
+
+  # If necessary, manually submit a CSR
+  # ignoring errors to simplify logic
+  run_task('peadm::submit_csr', $replica_target, _catch_errors => true)
+
+  # On primary, if necessary, sign the certificate request
+  run_task('peadm::sign_csr', $primary_target, certnames => [$replica_target.peadm::certname()] )
+
+  # On <replica_target>, run the puppet agent 
   run_task('peadm::puppet_runonce', $replica_target)
 
   # On the PE-PostgreSQL server in the <replacement-avail-group-letter> group
@@ -44,7 +60,7 @@
 
   #  pe-puppetdb-pe-puppetdb-map <replacement-replica-fqdn> pe-puppetdb
   #  pe-puppetdb-pe-puppetdb-migrator-map <replacement-replica-fqdn> pe-puppetdb-migrator
-  apply($replica_postgresql_target) {
+  apply($replica_postgresql_target, _catch_errors => true) {
     file_line { 'pe-puppetdb-pe-puppetdb-map':
       path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_ident.conf',
       line => "pe-puppetdb-pe-puppetdb-map ${replica_target.peadm::certname()} pe-puppetdb",
@@ -56,7 +72,7 @@
   }
 
   # Restart pe-postgresql.service
-  run_command('systemctl restart pe-postgresql.service', $replica_postgresql_target)
+  run_command('systemctl restart pe-postgresql.service', $replica_postgresql_target, _catch_errors => true)
 
   # Provision the new system as a replica
   run_task('peadm::provision_replica', $primary_target,
@@ -70,7 +86,7 @@
   )
 
   # start puppet service on postgresql host
-  run_command('systemctl stop puppet.service', $replica_postgresql_target)
+  run_command('systemctl start puppet.service', $replica_postgresql_target)
 
 
 }

Diff for: spec/fixtures/modules/peadm_spec/plans/add_replica.pp

@@ -0,0 +1,26 @@
+plan peadm_spec::add_replica(
+){
+
+  $t = get_targets('*')
+  wait_until_available($t)
+
+  parallelize($t) |$target| {
+    $fqdn = run_command('hostname -f', $target)
+    $target.set_var('certname', $fqdn.first['stdout'].chomp)
+  }
+
+  $primary_host = $t.filter |$n| { $n.vars['role'] == 'primary' }
+  $replica_host = $t.filter |$n| { $n.vars['role'] == 'replica' }
+  $replica_postgresql_host = $t.filter |$n| { $n.vars['role'] == 'replica-pdb-postgresql' }
+
+  if $replica_host == [] {
+    fail_plan('"replica" role missing from inventory, cannot continue')
+  }
+
+  run_plan('peadm::add_replica',
+    primary_host            => $primary_host,
+    replica_host            => $replica_host,
+    replica_postgresql_host => $replica_postgresql_host ? { [] => $replica_host, default => $replica_postgresql_host }
+  )
+
+}