From 66504c6f6f33b95d094fc5d6aefb61f106254065 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Wed, 20 Apr 2022 09:51:47 -0700
Subject: [PATCH 01/11] (SOLARCH-581) Implement restore plan

This commit holds WIP for the peadm::restore plan.
---
 plans/restore.pp                           | 204 +++++++++++++++++++++
 spec/plans/restore_spec.rb                 |  24 +++
 tasks/restore_classification.json          |  12 ++
 tasks/restore_classification.rb            |  45 +++++
 tasks/transform_classification_groups.json |  17 ++
 tasks/transform_classification_groups.py   |  79 ++++++++
 6 files changed, 381 insertions(+)
 create mode 100644 plans/restore.pp
 create mode 100644 spec/plans/restore_spec.rb
 create mode 100644 tasks/restore_classification.json
 create mode 100755 tasks/restore_classification.rb
 create mode 100644 tasks/transform_classification_groups.json
 create mode 100644 tasks/transform_classification_groups.py

diff --git a/plans/restore.pp b/plans/restore.pp
new file mode 100644
index 00000000..95ec4fa3
--- /dev/null
+++ b/plans/restore.pp
@@ -0,0 +1,204 @@
+# @summary Restore the core user settings for puppet infrastructure from backup
+#
+# This plan can restore data to puppet infrastructure for DR and rebuilds
+# 
+plan peadm::restore (
+  # Standard
+  Peadm::SingleTargetSpec           $primary_host,
+  # Which data to restore
+  Boolean                            $restore_orchestrator    = true,
+  Boolean                            $restore_rbac            = true,
+  Boolean                            $restore_activity        = true,
+  Boolean                            $restore_ca_ssl          = true,
+  Boolean                            $restore_puppetdb        = false,
+  Boolean                            $restore_classification  = true,
+  String                             $input_directory         = '/tmp',
+  String                             $working_directory       = '/tmp',
+  String                             $backup_timestamp,
+){
+  peadm::assert_supported_bolt_version()
+  $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value
+
+  $arch = peadm::assert_supported_architecture(
+    $primary_host,
+    $cluster['params']['replica_host'],
+    $cluster['params']['primary_postgresql_host'],
+    $cluster['params']['replica_postgresql_host'],
+    $cluster['params']['compiler_hosts'],
+  )
+  $servers = delete_undef_values([$primary_host , $cluster['params']['replica_host'] ])
+  $cluster_servers = delete_undef_values($servers + $cluster['params']['compiler_hosts'] + [ $cluster['params']['primary_postgresql_host'], $cluster['params']['replica_postgresql_host']]) # lint:ignore:140chars
+  if $cluster['params']['compiler_hosts'] {
+    $check_puppetdb_on_compilers = run_task('service', $cluster['params']['compiler_hosts'],
+      action => 'status',
+      name   => 'pe-puppetdb'
+    )
+    $puppetdb_on_compilers = $check_puppetdb_on_compilers.filter_set | $result | {
+      $result['enabled'] == 'enabled'
+    }.targets
+  } else {
+    $puppetdb_on_compilers = undef
+  }
+  $puppetdb_servers = delete_undef_values([$servers,$puppetdb_on_compilers])
+  $backup_directory = "${input_directory}/pe-backup-${backup_timestamp}"
+  $database_backup_directory = "${working_directory}/pe-backup-databases-${backup_timestamp}"
+  # I need the actual hostname for the certificate in a remote puppetdb backup. If a user sends primary host as IP it will fail
+  $primary_host_fqdn = $cluster['params']['primary_host']
+  $primary_postgresql_host = $cluster['params']['primary_postgresql_host']
+  apply($primary_host){
+    file { $database_backup_directory :
+      ensure => 'directory',
+      owner  => 'pe-puppetdb',
+      group  => 'pe-postgres',
+      mode   => '0770'
+    }
+  }
+
+  # Create an array of the names of databases and whether they have to be backed up to use in a lambda later
+  $database_to_restore = [ $restore_orchestrator, $restore_activity, $restore_rbac, $restore_puppetdb]
+  $database_names      = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ]
+
+  peadm::assert_supported_bolt_version()
+
+  if $restore_classification {
+
+    out::message('# Restoring classification')
+    run_task('peadm::backup_classification', $primary_host,
+      directory => $working_directory
+    )
+    out::message("# Backed up current classification to ${working_directory}/classification_backup.json")
+
+    run_task('peadm::transform_classification_groups', $primary_host,
+      source_directory => $backup_directory,
+      working_directory => $working_directory
+    )
+
+    run_task('peadm::restore_classification', $primary_host,
+    classification_file => "${working_directory}/classification_backup.json",
+    )
+  }
+
+  if $restore_ca_ssl {
+    out::message('# Restoring ca and ssl certificates')
+    run_command("/opt/puppetlabs/bin/puppet-backup restore ${backup_directory}/pe_backup-*tgz --scope=certs --tempdir=${working_directory} --force", $primary_host) # lint:ignore:140chars
+  }
+
+  ## shutdown services
+    run_task('service', $servers,
+      action => 'stop',
+      name   => 'pe-console-services'
+    )
+    run_task('service', $primary_host,
+      action => 'stop',
+      name   => 'pe-nginx'
+    )
+    run_task('service', $servers,
+      action => 'stop',
+      name   => 'pe-puppetserver'
+    )
+    run_task('service', $servers,
+      action => 'stop',
+      name   => 'pxp-agent'
+    )
+    run_task('service', $primary_host,
+      action => 'stop',
+      name   => 'pe-orchestration-services'
+    )
+    run_task('service', $cluster_servers,
+      action => 'stop',
+      name   => 'puppet'
+    )
+    run_task('service', $puppetdb_servers ,
+      action => 'stop',
+      name   => 'pe-puppetdb'
+    )
+
+
+  # Restore secrets/keys.json if it exists
+  out::message('# Restoring ldap secret key if it exists')
+  run_command("test -f ${backup_directory}//keys.json && cp -rp ${backup_directory}/keys.json /etc/puppetlabs/console-services/conf.d/secrets/ || echo secret ldap key doesnt exist" , $primary_host) # lint:ignore:140chars
+
+  # IF restoring orchestrator restore the secrets too /etc/puppetlabs/orchestration-services/conf.d/secrets/
+  if $restore_orchestrator {
+    out::message('# Restoring orchestrator secret keys')
+    run_command("cp -rp ${backup_directory}/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets ", $primary_host)
+  }
+
+  $database_to_restore.each |Integer $index, Boolean $value | {
+    if $value {
+    out::message("# Restoring database ${database_names[$index]}")
+      # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary.
+      if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host {
+        # Drop pglogical extensions and schema if present
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"", $primary_postgresql_host) # lint:ignore:140chars
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'\"", $primary_postgresql_host) # lint:ignore:140chars
+        # To allow pe-puppetdb to restore the database grant temporary privileges
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'ALTER USER \\\"pe-puppetdb\\\" WITH SUPERUSER;'\"", $primary_postgresql_host) # lint:ignore:140chars
+        # Restore database
+        run_command("/opt/puppetlabs/server/bin/pg_restore -d \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host_fqdn}.cert.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host_fqdn}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb user=pe-puppetdb\" -Fd ${backup_directory}/puppetdb_*" , $primary_host) # lint:ignore:140chars
+        # Remove pe-puppetdb privileges post restore
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'ALTER USER \\\"pe-puppetdb\\\" WITH NOSUPERUSER;'\"", $primary_postgresql_host) # lint:ignore:140chars
+        # Drop pglogical extension and schema (again) if present after db restore
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"",$primary_postgresql_host) # lint:ignore:140chars
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;;'\"",$primary_postgresql_host) # lint:ignore:140chars
+      } else {
+        # Drop pglogical extensions and schema if present
+        run_command("su - pe-postgres -s '/bin/bash' -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"", $primary_host) # lint:ignore:140chars
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'\"", $primary_host) # lint:ignore:140chars
+        # Restore database
+        run_command("cp -pr ${backup_directory}/${database_names[$index]}_* ${database_backup_directory}/ ", $primary_host )
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/pg_restore ${database_backup_directory}/${database_names[$index]}_* -Fd -j4 --dbname=${database_names[$index]}\"", $primary_host)# lint:ignore:140chars
+        run_command("sudo -H -u pe-postgres /opt/puppetlabs/server/bin/pg_restore -d ${database_names[$index]} -c ${database_backup_directory}/${database_names[$index]}_*",$primary_host) # lint:ignore:140chars
+        run_command("rm -rf ${database_backup_directory}/${database_names[$index]}_*", $primary_host )
+        # Drop pglogical extension and schema (again) if present after db restore
+        run_command("su - pe-postgres -s '/bin/bash' -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"",$primary_host) # lint:ignore:140chars
+        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;'\"",$primary_host) # lint:ignore:140chars
+      }
+    }
+  }
+
+  ## Restart services
+  run_task('service', $primary_host,
+    action => 'start',
+    name   => 'pe-orchestration-services'
+  )
+  run_task('service', $servers,
+    action => 'start',
+    name   => 'pxp-agent'
+  )
+  run_task('service', $servers,
+    action => 'start',
+    name   => 'pe-puppetserver'
+  )
+  run_task('service', $primary_host,
+    action => 'start',
+    name   => 'pe-nginx'
+  )
+  run_task('service', $servers,
+    action => 'start',
+    name   => 'pe-console-services'
+  )
+  run_task('service', $cluster_servers,
+    action => 'start',
+    name   => 'puppet'
+  )
+  run_task('service', $puppetdb_servers,
+    action => 'start',
+    name   => 'pe-puppetdb'
+  )
+# If we have replicas reinitalise any databases restored
+  if $cluster['params']['replica_host'] {
+    $database_to_restore.each |Integer $index, Boolean $value | {
+      if $database_names[$index] != 'pe-puppetdb' and $cluster['params']['replica_postgresql_host'] {
+        run_command("/opt/puppetlabs/bin/puppet-infra reinitialize replica --db ${database_names[$index]} -y", $cluster['params']['replica_host'] ) # lint:ignore:140chars
+      }
+    }
+  }
+
+  apply($primary_host){
+    file { $database_backup_directory :
+      ensure => 'absent',
+      force  => true
+    }
+  }
+}
diff --git a/spec/plans/restore_spec.rb b/spec/plans/restore_spec.rb
new file mode 100644
index 00000000..47763365
--- /dev/null
+++ b/spec/plans/restore_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe 'peadm::restore' do
+  include BoltSpec::Plans
+  let(:params) { { 'primary_host' => 'primary', 'backup_timestamp' => '2022-03-29_16:57:41' } }
+
+  it 'runs with default params' do
+    allow_apply
+    pending('a lack of support for functions requires a workaround to be written')
+    expect_task('peadm::get_peadm_config').always_return({ 'primary_postgresql_host' => 'postgres' })
+    expect_out_message.with_params('# Backing up ca and ssl certificates')
+    # The commands all have a timestamp in them and frankly its proved to hard with bolt spec to work this out
+    allow_any_command
+    expect_out_message.with_params('# Restoring classification')
+    expect_out_message.with_params('# Backed up current classification to /tmp/classification_backup.json')
+    expect_out_message.with_params('# Restoring ca and ssl certificates')
+    expect_out_message.with_params('# Restoring database pe-orchestrator')
+    expect_out_message.with_params('# Restoring database pe-activity')
+    expect_out_message.with_params('# Restoring database pe-rbac')
+    expect_out_message.with_params('# Restoring classification')
+    expect_task('peadm::backup_classification')
+    expect(run_plan('peadm::restore', params)).to be_ok
+  end
+end
diff --git a/tasks/restore_classification.json b/tasks/restore_classification.json
new file mode 100644
index 00000000..3f527ce3
--- /dev/null
+++ b/tasks/restore_classification.json
@@ -0,0 +1,12 @@
+{
+  "puppet_task_version": 1,
+  "supports_noop": false,
+  "description": "A short description of this task",
+  "parameters": {
+    "classification_file": {
+      "type": "String",
+      "description": "The full path to a backed up or transformed classification file",
+      "default": "/tmp/transformed_classification.json"
+    }
+  }
+}
diff --git a/tasks/restore_classification.rb b/tasks/restore_classification.rb
new file mode 100755
index 00000000..cf08a248
--- /dev/null
+++ b/tasks/restore_classification.rb
@@ -0,0 +1,45 @@
+#!/opt/puppetlabs/puppet/bin/ruby
+
+# Puppet Task Name: backup_classification
+require 'net/https'
+require 'uri'
+require 'json'
+require 'puppet'
+
+# RestoreClassifiation task class
+class RestoreClassification
+  def initialize(params)
+    @classification_file = params['classification_file']
+  end
+
+  def execute!
+    restore_classification
+    puts "Classification restored from #{@classification_file}"
+  end
+
+  private
+
+  def https_client
+    client = Net::HTTP.new('localhost', '4433')
+    client.use_ssl = true
+    client.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert]))
+    client.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey]))
+    client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    client
+  end
+
+  def restore_classification
+    classification = https_client
+    classification_post = Net::HTTP::Post.new('/classifier-api/v1/import-hierarchy', 'Content-Type' => 'application/json')
+    classification_post.body = File.read(@classification_file)
+    classification.request(classification_post)
+  end
+end
+# Run the task unless an environment flag has been set, signaling not to. The
+# environment flag is used to disable auto-execution and enable Ruby unit
+# testing of this task.
+unless ENV['RSPEC_UNIT_TEST_MODE']
+  Puppet.initialize_settings
+  task = RestoreClassification.new(JSON.parse(STDIN.read))
+  task.execute!
+end
diff --git a/tasks/transform_classification_groups.json b/tasks/transform_classification_groups.json
new file mode 100644
index 00000000..4284d04f
--- /dev/null
+++ b/tasks/transform_classification_groups.json
@@ -0,0 +1,17 @@
+{
+    "description": "Transform the user groups from a source backup to a list of groups on the target server",
+    "parameters": {
+      "source_directory": {
+        "type": "String",
+        "description": "Location of Source node group yaml file"
+      },
+      "working_directory": {
+        "type": "String",
+        "description": "Location of target node group yaml file and where to create the transformed file"
+      }
+    },
+    "input_method": "stdin",
+    "implementations": [
+      {"name": "transform_classification_groups.py"}
+    ]
+  }
\ No newline at end of file
diff --git a/tasks/transform_classification_groups.py b/tasks/transform_classification_groups.py
new file mode 100644
index 00000000..ca5355b7
--- /dev/null
+++ b/tasks/transform_classification_groups.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+"""This module takes two classification outputs from source and targer puppet infrastructure and
+takes the user defintions from the source and adds them to the infrastructure defintions of the
+target. Allowing the ability to restore a backup of user node definitions"""
+
+import json
+import sys
+params = json.load(sys.stdin)
+source_classification_file      = params['source_directory']+"/classification_backup.json"
+target_classification_file      = params['working_directory']+"/classification_backup.json"
+transformed_classification_file = params['working_directory']+"/transformed_classification.json"
+
+def removesubgroups(data_rsg,id_rsg):
+    """
+    This definition allows us to traverse recursively down the json groups finding all children of
+    the pe infrastructure and to remove them.
+
+    Inputs are the resource group and parent ID of the resource groups
+
+    Returns
+    -------
+    data_rsg : list
+       The resource groups which did not have the parent ID
+    """
+    groups = list(filter(lambda x:x ["parent"]==id_rsg,data_rsg))
+    for group in groups:
+        subid = group["id"]
+        data_rsg = list(filter(lambda x:x ["id"]!=subid,data_rsg)) # pylint: disable=cell-var-from-loop
+        data_rsg = removesubgroups(data_rsg,subid)
+    return data_rsg
+
+# This defintion allows us to traverse down the pe inf tree and find all groups
+def addsubgroups(data_asg,id_asg,peinf_asg):
+    """
+    This definition allows us to traverse recursively down the json groups finding all groups in
+    the pe infrastructure tree and adding them to a list recursively and then returning the list.
+
+    Inputs are the list of all resource groups, infrastructure resource groups found so far and
+    parent ID of infrastructure groups
+
+    Returns
+    -------
+    data_asg : list
+       The list of resource groups of pe infrastructure groups at source
+    """
+    groups = list(filter(lambda x:x ["parent"]==id_asg,data_asg))
+    peinf_asg = peinf_asg + groups
+    for group in groups:
+        subid = group["id"]
+        peinf_asg = addsubgroups(data_asg,subid,peinf_asg)
+    return peinf_asg
+
+# open the backup classification
+with open(source_classification_file) as data_file:
+    data = json.load(data_file)
+# open the DR server classification
+with open(target_classification_file) as data_fileDR:
+    data_DR = json.load(data_fileDR)
+
+
+# find the infrastructure group and its ID
+peinf = list(filter(lambda x:x ["name"]=="PE Infrastructure",data))
+group_id = peinf[0]["id"]
+# remove this group from the list and recursively remove all sub groups
+data = list(filter(lambda x:x ["id"]!=group_id,data))
+data = removesubgroups(data,group_id)
+
+# find the dr infrastructure group and its ID
+peinf_DR = list(filter(lambda x:x ["name"]=="PE Infrastructure",data_DR))
+id_DR = peinf_DR[0]["id"]
+# Recursively go through inf groups to get the full tree
+peinf_DR = addsubgroups(data_DR,id_DR,peinf_DR)
+
+# Add the contents of the backup classification without pe inf to the DR pe inf groups
+# and write to a file
+peinf_transformed_groups = data + peinf_DR
+with open(transformed_classification_file, 'w') as fp:
+    json.dump(peinf_transformed_groups, fp)
+    
\ No newline at end of file

From 7bf4165cb3d06db28dba03c047b30bd846bbdded Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Thu, 28 Apr 2022 16:07:15 -0700
Subject: [PATCH 02/11] Refactor restoer plan for style and flow

This commit refactors the restore plan for style (line folding, variable
names, etc) and flow, mainly an attempt to find a single path that
always works, and eliminate conditional logic.

The plan is not yet validated at this stage. More work is required to
ensure correct functionality.
---
 plans/restore.pp | 310 +++++++++++++++++++++++------------------------
 1 file changed, 152 insertions(+), 158 deletions(-)

diff --git a/plans/restore.pp b/plans/restore.pp
index 95ec4fa3..d45dab4e 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -3,202 +3,196 @@
 # This plan can restore data to puppet infrastructure for DR and rebuilds
 # 
 plan peadm::restore (
-  # Standard
-  Peadm::SingleTargetSpec           $primary_host,
+  # This plan should be run on the primary server
+  Peadm::SingleTargetSpec $targets,
+
   # Which data to restore
-  Boolean                            $restore_orchestrator    = true,
-  Boolean                            $restore_rbac            = true,
-  Boolean                            $restore_activity        = true,
-  Boolean                            $restore_ca_ssl          = true,
-  Boolean                            $restore_puppetdb        = false,
-  Boolean                            $restore_classification  = true,
-  String                             $input_directory         = '/tmp',
-  String                             $working_directory       = '/tmp',
-  String                             $backup_timestamp,
-){
+  Peadm::Recovery_opts    $restore = {},
+
+  # Path to the recovery tarball
+  Pattern[/.*\.tar\.gz$/] $input_file,
+) {
   peadm::assert_supported_bolt_version()
-  $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value
 
+  $recovery_opts = (peadm::recovery_opts_default() + $restore)
+  $cluster = run_task('peadm::get_peadm_config', $targets).first.value
   $arch = peadm::assert_supported_architecture(
-    $primary_host,
-    $cluster['params']['replica_host'],
-    $cluster['params']['primary_postgresql_host'],
-    $cluster['params']['replica_postgresql_host'],
-    $cluster['params']['compiler_hosts'],
+    getvar('cluster.params.primary_host'),
+    getvar('cluster.params.replica_host'),
+    getvar('cluster.params.primary_postgresql_host'),
+    getvar('cluster.params.replica_postgresql_host'),
+    getvar('cluster.params.compiler_hosts'),
   )
-  $servers = delete_undef_values([$primary_host , $cluster['params']['replica_host'] ])
-  $cluster_servers = delete_undef_values($servers + $cluster['params']['compiler_hosts'] + [ $cluster['params']['primary_postgresql_host'], $cluster['params']['replica_postgresql_host']]) # lint:ignore:140chars
-  if $cluster['params']['compiler_hosts'] {
-    $check_puppetdb_on_compilers = run_task('service', $cluster['params']['compiler_hosts'],
-      action => 'status',
-      name   => 'pe-puppetdb'
-    )
-    $puppetdb_on_compilers = $check_puppetdb_on_compilers.filter_set | $result | {
-      $result['enabled'] == 'enabled'
-    }.targets
-  } else {
-    $puppetdb_on_compilers = undef
+
+  $primary_target   = peadm::get_targets(getvar('cluster.params.primary_host'), 1)
+  $replica_target   = peadm::get_targets(getvar('cluster.params.replica_host'), 1)
+  $compiler_targets = peadm::get_targets(getvar('cluster.params.compiler_hosts'))
+  $puppetdb_postgresql_target = getvar('cluster.params.primary_postgresql_host') ? {
+    undef   => $primary_target,
+    default => peadm::get_targets(getvar('cluster.params.primary_postgresql_host'), 1),
   }
-  $puppetdb_servers = delete_undef_values([$servers,$puppetdb_on_compilers])
-  $backup_directory = "${input_directory}/pe-backup-${backup_timestamp}"
+
+  $puppetdb_targets = peadm::flatten_compact([
+    $primary_target,
+    $replica_target,
+    $compiler_targets,
+  ])
+
+  $recovery_directory = "${dirname($input_file)}/${basename("${input_file}", '.tar.gz')}"
   $database_backup_directory = "${working_directory}/pe-backup-databases-${backup_timestamp}"
-  # I need the actual hostname for the certificate in a remote puppetdb backup. If a user sends primary host as IP it will fail
-  $primary_host_fqdn = $cluster['params']['primary_host']
-  $primary_postgresql_host = $cluster['params']['primary_postgresql_host']
-  apply($primary_host){
-    file { $database_backup_directory :
-      ensure => 'directory',
-      owner  => 'pe-puppetdb',
-      group  => 'pe-postgres',
-      mode   => '0770'
-    }
-  }
+
+  run_command(@("CMD"/L), $primary_target)
+    umask 0077 \
+      && cd ${shellquote($recovery_directory)} \
+      && tar -xzf ${shellquote($input_file)}
+    | CMD
 
   # Create an array of the names of databases and whether they have to be backed up to use in a lambda later
   $database_to_restore = [ $restore_orchestrator, $restore_activity, $restore_rbac, $restore_puppetdb]
   $database_names      = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ]
 
-  peadm::assert_supported_bolt_version()
-
-  if $restore_classification {
+  $restore_databases = {
+    'orchestrator' => $primary_target,
+    'activity'     => $primary_target,
+    'rbac'         => $primary_target,
+    'puppetdb'     => $puppetdb_postgresql_target,
+  }.filter |$key,$_| {
+    $recovery_opts[$key] == true
+  }
 
+  if getvar('recovery_opts.classifier') {
     out::message('# Restoring classification')
-    run_task('peadm::backup_classification', $primary_host,
-      directory => $working_directory
+    run_task('peadm::backup_classification', $primary_target,
+      directory => $recovery_directory
     )
-    out::message("# Backed up current classification to ${working_directory}/classification_backup.json")
+    out::message("# Backed up current classification to ${recovery_directory}/classification_backup.json")
 
-    run_task('peadm::transform_classification_groups', $primary_host,
-      source_directory => $backup_directory,
-      working_directory => $working_directory
+    run_task('peadm::transform_classification_groups', $primary_target,
+      source_directory  => "${recovery_directory}/classifier",
+      working_directory => $recovery_directory
     )
 
-    run_task('peadm::restore_classification', $primary_host,
-    classification_file => "${working_directory}/classification_backup.json",
+    run_task('peadm::restore_classification', $primary_target,
+      classification_file => "${recovery_directory}/classification_backup.json",
     )
   }
 
-  if $restore_ca_ssl {
+  if getvar('recovery_opts.ca') {
     out::message('# Restoring ca and ssl certificates')
-    run_command("/opt/puppetlabs/bin/puppet-backup restore ${backup_directory}/pe_backup-*tgz --scope=certs --tempdir=${working_directory} --force", $primary_host) # lint:ignore:140chars
+    run_command(@("CMD"/L), $primary_target)
+      /opt/puppetlabs/bin/puppet-backup restore \
+        --scope=certs \
+        --tempdir=${shellquote($recovery_directory)} \
+        --force \
+        ${shellquote($recovery_directory)}/classifier/pe_backup-*tgz
+      | CMD
   }
 
   ## shutdown services
-    run_task('service', $servers,
-      action => 'stop',
-      name   => 'pe-console-services'
-    )
-    run_task('service', $primary_host,
-      action => 'stop',
-      name   => 'pe-nginx'
-    )
-    run_task('service', $servers,
-      action => 'stop',
-      name   => 'pe-puppetserver'
-    )
-    run_task('service', $servers,
-      action => 'stop',
-      name   => 'pxp-agent'
-    )
-    run_task('service', $primary_host,
-      action => 'stop',
-      name   => 'pe-orchestration-services'
-    )
-    run_task('service', $cluster_servers,
-      action => 'stop',
-      name   => 'puppet'
-    )
-    run_task('service', $puppetdb_servers ,
-      action => 'stop',
-      name   => 'pe-puppetdb'
-    )
-
+  run_command(@("CMD"/L), $primary_target)
+    systemctl stop pe-console-services pe-nginx pxp-agent pe-puppetserver \
+                   pe-orchestration-services puppet pe-puppetdb
+    | CMD
 
   # Restore secrets/keys.json if it exists
   out::message('# Restoring ldap secret key if it exists')
-  run_command("test -f ${backup_directory}//keys.json && cp -rp ${backup_directory}/keys.json /etc/puppetlabs/console-services/conf.d/secrets/ || echo secret ldap key doesnt exist" , $primary_host) # lint:ignore:140chars
+  run_command(@("CMD"/L), $primary_target)
+    test -f ${shellquote($backup_directory)}/rbac/keys.json \
+      && cp -rp ${shellquote($backup_directory)}/keys.json /etc/puppetlabs/console-services/conf.d/secrets/ \
+      || echo secret ldap key doesnt exist
+    | CMD
 
-  # IF restoring orchestrator restore the secrets too /etc/puppetlabs/orchestration-services/conf.d/secrets/
-  if $restore_orchestrator {
+  # IF restoring orchestrator restore the secrets to /etc/puppetlabs/orchestration-services/conf.d/secrets/
+  if getvar('recovery_opts.orchestrator') {
     out::message('# Restoring orchestrator secret keys')
-    run_command("cp -rp ${backup_directory}/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets ", $primary_host)
+    run_command(@("CMD"/L), $primary_target)
+      cp -rp ${shellquote($backup_directory)}/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets/
+      | CMD
   }
 
-  $database_to_restore.each |Integer $index, Boolean $value | {
-    if $value {
-    out::message("# Restoring database ${database_names[$index]}")
-      # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary.
-      if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host {
-        # Drop pglogical extensions and schema if present
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"", $primary_postgresql_host) # lint:ignore:140chars
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'\"", $primary_postgresql_host) # lint:ignore:140chars
-        # To allow pe-puppetdb to restore the database grant temporary privileges
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'ALTER USER \\\"pe-puppetdb\\\" WITH SUPERUSER;'\"", $primary_postgresql_host) # lint:ignore:140chars
-        # Restore database
-        run_command("/opt/puppetlabs/server/bin/pg_restore -d \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host_fqdn}.cert.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host_fqdn}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb user=pe-puppetdb\" -Fd ${backup_directory}/puppetdb_*" , $primary_host) # lint:ignore:140chars
-        # Remove pe-puppetdb privileges post restore
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'ALTER USER \\\"pe-puppetdb\\\" WITH NOSUPERUSER;'\"", $primary_postgresql_host) # lint:ignore:140chars
-        # Drop pglogical extension and schema (again) if present after db restore
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"",$primary_postgresql_host) # lint:ignore:140chars
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;;'\"",$primary_postgresql_host) # lint:ignore:140chars
-      } else {
-        # Drop pglogical extensions and schema if present
-        run_command("su - pe-postgres -s '/bin/bash' -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"", $primary_host) # lint:ignore:140chars
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'\"", $primary_host) # lint:ignore:140chars
-        # Restore database
-        run_command("cp -pr ${backup_directory}/${database_names[$index]}_* ${database_backup_directory}/ ", $primary_host )
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/pg_restore ${database_backup_directory}/${database_names[$index]}_* -Fd -j4 --dbname=${database_names[$index]}\"", $primary_host)# lint:ignore:140chars
-        run_command("sudo -H -u pe-postgres /opt/puppetlabs/server/bin/pg_restore -d ${database_names[$index]} -c ${database_backup_directory}/${database_names[$index]}_*",$primary_host) # lint:ignore:140chars
-        run_command("rm -rf ${database_backup_directory}/${database_names[$index]}_*", $primary_host )
-        # Drop pglogical extension and schema (again) if present after db restore
-        run_command("su - pe-postgres -s '/bin/bash' -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"",$primary_host) # lint:ignore:140chars
-        run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;'\"",$primary_host) # lint:ignore:140chars
-      }
-    }
-  }
+  #$database_to_restore.each |Integer $index, Boolean $value | {
+  $restore_databases.each |$name,$database_target| {
+    out::message("# Restoring ${name} database")
+    $dbname = "pe-${shellquote($name)}"
 
-  ## Restart services
-  run_task('service', $primary_host,
-    action => 'start',
-    name   => 'pe-orchestration-services'
-  )
-  run_task('service', $servers,
-    action => 'start',
-    name   => 'pxp-agent'
-  )
-  run_task('service', $servers,
-    action => 'start',
-    name   => 'pe-puppetserver'
-  )
-  run_task('service', $primary_host,
-    action => 'start',
-    name   => 'pe-nginx'
-  )
-  run_task('service', $servers,
-    action => 'start',
-    name   => 'pe-console-services'
-  )
-  run_task('service', $cluster_servers,
-    action => 'start',
-    name   => 'puppet'
-  )
-  run_task('service', $puppetdb_servers,
-    action => 'start',
-    name   => 'pe-puppetdb'
-  )
-# If we have replicas reinitalise any databases restored
-  if $cluster['params']['replica_host'] {
-    $database_to_restore.each |Integer $index, Boolean $value | {
-      if $database_names[$index] != 'pe-puppetdb' and $cluster['params']['replica_postgresql_host'] {
-        run_command("/opt/puppetlabs/bin/puppet-infra reinitialize replica --db ${database_names[$index]} -y", $cluster['params']['replica_host'] ) # lint:ignore:140chars
-      }
-    }
+    # Drop pglogical extensions and schema if present
+    run_command(@("CMD"/L), $database_target)
+      su - pe-postgres -s /bin/bash -c \
+        "/opt/puppetlabs/server/bin/psql \
+           --tuples-only \
+           -d '${dbname}}' \
+           -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'"
+      | CMD
+
+    run_command(@("CMD"/L), $database_target)
+      su - pe-postgres -s /bin/bash -c \
+        "/opt/puppetlabs/server/bin/psql \
+           -d '${dbname}' \
+           -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'"
+      | CMD
+
+    # To allow db user to restore the database grant temporary privileges
+    run_command(@("CMD"/L), $database_target)
+      su - pe-postgres -s /bin/bash -c \
+        "/opt/puppetlabs/server/bin/psql \
+           -d '${dbname}' \
+           -c 'ALTER USER \"${dbname}\" WITH SUPERUSER;'"
+      | CMD
+
+    # Restore database
+    run_command(@("CMD"/L), $primary_target)
+      /opt/puppetlabs/server/bin/pg_restore \
+        -d "sslmode=verify-ca \
+            host=${shellquote($database_target.peadm::certname())} \
+            sslcert=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.cert.pem \
+            sslkey=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.private_key.pem \
+            sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem \
+            dbname=${dbname} \
+            user=${dbname}" \
+        -Fd ${backup_directory}/${name}/${dbname}.dump.d
+      | CMD
+
+    # Remove db user privileges post restore
+    run_command(@("CMD"/L), $database_target)
+      su - pe-postgres -s /bin/bash -c \
+        "/opt/puppetlabs/server/bin/psql \
+           -d '${dbname}' \
+           -c 'ALTER USER \"${dbname}\" WITH NOSUPERUSER;'"
+      | CMD
+
+    # Drop pglogical extension and schema (again) if present after db restore
+    run_command(@("CMD"/L), $database_target)
+      su - pe-postgres -s /bin/bash -c \
+        "/opt/puppetlabs/server/bin/psql \
+           --tuples-only \
+           -d '${dbname}' \
+           -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'"
+      | CMD
+
+    run_command(@("CMD"/L), $database_target)
+      su - pe-postgres -s /bin/bash -c \
+        "/opt/puppetlabs/server/bin/psql \
+           -d '${dbname}' \
+           -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;'"
+      | CMD
   }
 
+  run_command(@("CMD"/L), $primary_target)
+    systemctl start pe-console-services pe-nginx pxp-agent pe-puppetserver \
+                    pe-orchestration-services puppet pe-puppetdb
+    | CMD
+
+  # If we have replicas reinitalise them
+  run_command(@("CMD"/L), $replica_target)
+    /opt/puppetlabs/bin/puppet-infra reinitialize replica -y
+    | CMD
+
   apply($primary_host){
-    file { $database_backup_directory :
+    file { $recovery_directory :
       ensure => 'absent',
       force  => true
     }
   }
+
+  return("success")
 }

From 9923b725bed88bc5cd7d308ae1690221c43d3f02 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Fri, 29 Apr 2022 15:17:28 -0700
Subject: [PATCH 03/11] Fixup restore plan

WIP: doing testing/validation and fixing logic
---
 plans/backup.pp  |  3 ++-
 plans/restore.pp | 19 +++++++------------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/plans/backup.pp b/plans/backup.pp
index 977457be..fa30add4 100644
--- a/plans/backup.pp
+++ b/plans/backup.pp
@@ -110,7 +110,8 @@
 
   run_command(@("CMD"/L), $primary_target)
     umask 0077 \
-      && tar -czf ${shellquote($backup_directory)}.tar.gz ${shellquote($backup_directory)} \
+      && cd ${shellquote(dirname($backup_directory))} \
+      && tar -czf ${shellquote($backup_directory)}.tar.gz ${shellquote(basename($backup_directory))} \
       && rm -rf ${shellquote($backup_directory)}
     | CMD
 
diff --git a/plans/restore.pp b/plans/restore.pp
index d45dab4e..8a20805a 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -39,18 +39,13 @@
   ])
 
   $recovery_directory = "${dirname($input_file)}/${basename("${input_file}", '.tar.gz')}"
-  $database_backup_directory = "${working_directory}/pe-backup-databases-${backup_timestamp}"
 
   run_command(@("CMD"/L), $primary_target)
     umask 0077 \
-      && cd ${shellquote($recovery_directory)} \
+      && cd ${shellquote(dirname($recovery_directory))} \
       && tar -xzf ${shellquote($input_file)}
     | CMD
 
-  # Create an array of the names of databases and whether they have to be backed up to use in a lambda later
-  $database_to_restore = [ $restore_orchestrator, $restore_activity, $restore_rbac, $restore_puppetdb]
-  $database_names      = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ]
-
   $restore_databases = {
     'orchestrator' => $primary_target,
     'activity'     => $primary_target,
@@ -97,8 +92,8 @@
   # Restore secrets/keys.json if it exists
   out::message('# Restoring ldap secret key if it exists')
   run_command(@("CMD"/L), $primary_target)
-    test -f ${shellquote($backup_directory)}/rbac/keys.json \
-      && cp -rp ${shellquote($backup_directory)}/keys.json /etc/puppetlabs/console-services/conf.d/secrets/ \
+    test -f ${shellquote($recovery_directory)}/rbac/keys.json \
+      && cp -rp ${shellquote($recovery_directory)}/keys.json /etc/puppetlabs/console-services/conf.d/secrets/ \
       || echo secret ldap key doesnt exist
     | CMD
 
@@ -106,7 +101,7 @@
   if getvar('recovery_opts.orchestrator') {
     out::message('# Restoring orchestrator secret keys')
     run_command(@("CMD"/L), $primary_target)
-      cp -rp ${shellquote($backup_directory)}/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets/
+      cp -rp ${shellquote($recovery_directory)}/orchestrator/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets/
       | CMD
   }
 
@@ -120,7 +115,7 @@
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            --tuples-only \
-           -d '${dbname}}' \
+           -d '${dbname}' \
            -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'"
       | CMD
 
@@ -149,7 +144,7 @@
             sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem \
             dbname=${dbname} \
             user=${dbname}" \
-        -Fd ${backup_directory}/${name}/${dbname}.dump.d
+        -Fd ${recovery_directory}/${name}/${dbname}.dump.d
       | CMD
 
     # Remove db user privileges post restore
@@ -187,7 +182,7 @@
     /opt/puppetlabs/bin/puppet-infra reinitialize replica -y
     | CMD
 
-  apply($primary_host){
+  apply($primary_target){
     file { $recovery_directory :
       ensure => 'absent',
       force  => true

From 9c1f7117837a5bf4fad6a8a24f00e809adf2b4bb Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Mon, 2 May 2022 08:31:11 -0700
Subject: [PATCH 04/11] Attempt to make restore PuppetDB work on replicas

Because the PuppetDB database exists separately on two PostgreSQL hosts
in a DR configuration, adapt the restore plan to restore the data to
BOTH locations, if they exist
---
 plans/restore.pp | 69 +++++++++++++++++++++++++++++-------------------
 1 file changed, 42 insertions(+), 27 deletions(-)

diff --git a/plans/restore.pp b/plans/restore.pp
index 8a20805a..45383096 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -27,10 +27,20 @@
   $primary_target   = peadm::get_targets(getvar('cluster.params.primary_host'), 1)
   $replica_target   = peadm::get_targets(getvar('cluster.params.replica_host'), 1)
   $compiler_targets = peadm::get_targets(getvar('cluster.params.compiler_hosts'))
-  $puppetdb_postgresql_target = getvar('cluster.params.primary_postgresql_host') ? {
-    undef   => $primary_target,
-    default => peadm::get_targets(getvar('cluster.params.primary_postgresql_host'), 1),
-  }
+
+  # Determine the array of targets to which the PuppetDB PostgreSQL database
+  # should be restored to. This could be as simple as just the primary server,
+  # or it could be two separate PostgreSQL servers.
+  $puppetdb_postgresql_targets = peadm::flatten_compact([
+    getvar('cluster.params.primary_postgresql_host') ? {
+      undef   => $primary_target,
+      default => peadm::get_targets(getvar('cluster.params.primary_postgresql_host'), 1),
+    },
+    getvar('cluster.params.replica_postgresql_host') ? {
+      undef   => $replica_target,
+      default => peadm::get_targets(getvar('cluster.params.replica_postgresql_host'), 1),
+    },
+  ])
 
   $puppetdb_targets = peadm::flatten_compact([
     $primary_target,
@@ -46,11 +56,13 @@
       && tar -xzf ${shellquote($input_file)}
     | CMD
 
+  # Map of recovery option name to array of database hosts to restore the
+  # relevant .dump content to.
   $restore_databases = {
-    'orchestrator' => $primary_target,
-    'activity'     => $primary_target,
-    'rbac'         => $primary_target,
-    'puppetdb'     => $puppetdb_postgresql_target,
+    'orchestrator' => [$primary_target],
+    'activity'     => [$primary_target],
+    'rbac'         => [$primary_target],
+    'puppetdb'     => $puppetdb_postgresql_targets,
   }.filter |$key,$_| {
     $recovery_opts[$key] == true
   }
@@ -106,12 +118,12 @@
   }
 
   #$database_to_restore.each |Integer $index, Boolean $value | {
-  $restore_databases.each |$name,$database_target| {
+  $restore_databases.each |$name,$database_targets| {
     out::message("# Restoring ${name} database")
     $dbname = "pe-${shellquote($name)}"
 
     # Drop pglogical extensions and schema if present
-    run_command(@("CMD"/L), $database_target)
+    run_command(@("CMD"/L), $database_targets)
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            --tuples-only \
@@ -119,7 +131,7 @@
            -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'"
       | CMD
 
-    run_command(@("CMD"/L), $database_target)
+    run_command(@("CMD"/L), $database_targets)
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            -d '${dbname}' \
@@ -127,28 +139,31 @@
       | CMD
 
     # To allow db user to restore the database grant temporary privileges
-    run_command(@("CMD"/L), $database_target)
+    run_command(@("CMD"/L), $database_targets)
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            -d '${dbname}' \
            -c 'ALTER USER \"${dbname}\" WITH SUPERUSER;'"
       | CMD
 
-    # Restore database
-    run_command(@("CMD"/L), $primary_target)
-      /opt/puppetlabs/server/bin/pg_restore \
-        -d "sslmode=verify-ca \
-            host=${shellquote($database_target.peadm::certname())} \
-            sslcert=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.cert.pem \
-            sslkey=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.private_key.pem \
-            sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem \
-            dbname=${dbname} \
-            user=${dbname}" \
-        -Fd ${recovery_directory}/${name}/${dbname}.dump.d
-      | CMD
+    # Restore database. If there are multiple database restore targets, perform
+    # the restore(s) in parallel.
+    parallelize($database_targets) |$database_target| {
+      run_command(@("CMD"/L), $primary_target)
+        /opt/puppetlabs/server/bin/pg_restore \
+          -d "sslmode=verify-ca \
+              host=${shellquote($database_target.peadm::certname())} \
+              sslcert=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.cert.pem \
+              sslkey=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.private_key.pem \
+              sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem \
+              dbname=${dbname} \
+              user=${dbname}" \
+          -Fd ${recovery_directory}/${name}/${dbname}.dump.d
+        | CMD
+    }
 
     # Remove db user privileges post restore
-    run_command(@("CMD"/L), $database_target)
+    run_command(@("CMD"/L), $database_targets)
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            -d '${dbname}' \
@@ -156,7 +171,7 @@
       | CMD
 
     # Drop pglogical extension and schema (again) if present after db restore
-    run_command(@("CMD"/L), $database_target)
+    run_command(@("CMD"/L), $database_targets)
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            --tuples-only \
@@ -164,7 +179,7 @@
            -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'"
       | CMD
 
-    run_command(@("CMD"/L), $database_target)
+    run_command(@("CMD"/L), $database_targets)
       su - pe-postgres -s /bin/bash -c \
         "/opt/puppetlabs/server/bin/psql \
            -d '${dbname}' \

From ff3ed7a6c1869516156633665ff20ebbbe52b2c1 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Mon, 2 May 2022 12:14:16 -0700
Subject: [PATCH 05/11] Make get_peadm_config more robust

Specifically, make it handle situations where PuppetDB queries return
multiple results for server() queries, which could happen in the event a
database has been backed up on one cluster, and restored into another.
---
 tasks/get_peadm_config.rb | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/tasks/get_peadm_config.rb b/tasks/get_peadm_config.rb
index e215649c..4193d0ad 100755
--- a/tasks/get_peadm_config.rb
+++ b/tasks/get_peadm_config.rb
@@ -18,13 +18,19 @@ def config
     # Compute values
     primary = groups.pinned('PE Master')
     replica = groups.pinned('PE HA Replica')
-    server_a = server('puppet/server', 'A')
-    server_b = server('puppet/server', 'B')
+    server_a = server('puppet/server', 'A', [primary, replica].compact)
+    server_b = server('puppet/server', 'B', [primary, replica].compact)
     primary_letter = primary.eql?(server_a) ? 'A' : 'B'
     replica_letter = primary_letter.eql?('A') ? 'B' : 'A'
+
+    configured_postgresql_servers = [
+      groups.dig("PE Primary A", 'config_data', 'puppet_enterprise::profile::puppetdb', 'database_host'),
+      groups.dig("PE Primary B", 'config_data', 'puppet_enterprise::profile::puppetdb', 'database_host'),
+    ].compact
+
     postgresql = {
-      'A' => server('puppet/puppetdb-database', 'A'),
-      'B' => server('puppet/puppetdb-database', 'B'),
+      'A' => server('puppet/puppetdb-database', 'A', configured_postgresql_servers),
+      'B' => server('puppet/puppetdb-database', 'B', configured_postgresql_servers),
     }
 
     # Build and return the task output
@@ -78,10 +84,11 @@ def compilers
     end
   end
 
-  def server(role, letter)
+  def server(role, letter, certname_array)
     query = 'inventory[certname] { '\
             '  trusted.extensions."1.3.6.1.4.1.34380.1.1.9812" = "' + role + '" and ' \
-            '  trusted.extensions."1.3.6.1.4.1.34380.1.1.9813" = "' + letter + '"}'
+            '  trusted.extensions."1.3.6.1.4.1.34380.1.1.9813" = "' + letter + '" and ' \
+            '  certname in ' + certname_array.to_json + '}'
 
     server = pdb_query(query).map { |n| n['certname'] }
     raise "More than one #{letter} #{role} server found!" unless server.size <= 1

From 74d3a999e6b5d515fb023d4bd4d610c6844caaed Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Mon, 2 May 2022 12:38:48 -0700
Subject: [PATCH 06/11] Add TODO's to peadm::restore

Note what additional action needs to take place when there is dev time
---
 plans/restore.pp | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/plans/restore.pp b/plans/restore.pp
index 45383096..c160d158 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -117,6 +117,9 @@
       | CMD
   }
 
+  # TODO: Use PuppetDB's /pdb/admin/v1/archive API to SAVE data currently in
+  #       PuppetDB. Otherwise we'll completely lose it if/when we restore.
+
   #$database_to_restore.each |Integer $index, Boolean $value | {
   $restore_databases.each |$name,$database_targets| {
     out::message("# Restoring ${name} database")
@@ -187,9 +190,14 @@
       | CMD
   }
 
+  # TODO: Use PuppetDB's /pdb/admin/v1/archive API to MERGE previously saved
+  #       data into the restored database.
+
+  # Use `puppet infra` to ensure correct file permissions, restart services,
+  # etc. Make sure not to try and get config data from the classifier, which
+  # isn't yet up and running.
   run_command(@("CMD"/L), $primary_target)
-    systemctl start pe-console-services pe-nginx pxp-agent pe-puppetserver \
-                    pe-orchestration-services puppet pe-puppetdb
+    /opt/puppetlabs/bin/puppet-infrastructure configure --no-recover
     | CMD
 
   # If we have replicas reinitalise them

From c5bb51edc30f3db582b8edf2a959370ae74aeeb3 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Mon, 2 May 2022 15:43:09 -0700
Subject: [PATCH 07/11] Add puppetdb export/import step to restore

This should help keep current cluster PuppetDB knowledge of cluster
hosts after restoring PuppetDB data from a backup
---
 plans/restore.pp | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/plans/restore.pp b/plans/restore.pp
index c160d158..b324126e 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -95,6 +95,16 @@
       | CMD
   }
 
+  # Use PuppetDB's /pdb/admin/v1/archive API to SAVE data currently in PuppetDB.
+  # Otherwise we'll completely lose it if/when we restore.
+  # TODO: consider adding a heuristic to skip when innappropriate due to size
+  #       or other factors.
+  if getvar('recovery_opts.puppetdb') {
+    run_command(@("CMD"/L), $primary_target)
+      /opt/puppetlabs/bin/puppet-db export ${shellquote($recovery_directory)}/puppetdb-archive.bin
+      | CMD
+  }
+
   ## shutdown services
   run_command(@("CMD"/L), $primary_target)
     systemctl stop pe-console-services pe-nginx pxp-agent pe-puppetserver \
@@ -117,9 +127,6 @@
       | CMD
   }
 
-  # TODO: Use PuppetDB's /pdb/admin/v1/archive API to SAVE data currently in
-  #       PuppetDB. Otherwise we'll completely lose it if/when we restore.
-
   #$database_to_restore.each |Integer $index, Boolean $value | {
   $restore_databases.each |$name,$database_targets| {
     out::message("# Restoring ${name} database")
@@ -190,8 +197,15 @@
       | CMD
   }
 
-  # TODO: Use PuppetDB's /pdb/admin/v1/archive API to MERGE previously saved
-  #       data into the restored database.
+  # Use PuppetDB's /pdb/admin/v1/archive API to MERGE previously saved data
+  # into the restored database.
+  # TODO: consider adding a heuristic to skip when innappropriate due to size
+  #       or other factors.
+  if getvar('recovery_opts.puppetdb') {
+    run_command(@("CMD"/L), $primary_target)
+      /opt/puppetlabs/bin/puppet-db import ${shellquote($recovery_directory)}/puppetdb-archive.bin
+      | CMD
+  }
 
   # Use `puppet infra` to ensure correct file permissions, restart services,
   # etc. Make sure not to try and get config data from the classifier, which

From 80e675405bc1a6c582fc58a421703473d6926647 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Mon, 2 May 2022 15:48:36 -0700
Subject: [PATCH 08/11] Use pg_restore with -j4 parallel

---
 plans/restore.pp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plans/restore.pp b/plans/restore.pp
index b324126e..cf7a5a98 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -161,6 +161,7 @@
     parallelize($database_targets) |$database_target| {
       run_command(@("CMD"/L), $primary_target)
         /opt/puppetlabs/server/bin/pg_restore \
+          -j 4 \
           -d "sslmode=verify-ca \
               host=${shellquote($database_target.peadm::certname())} \
               sslcert=/etc/puppetlabs/puppetdb/ssl/${shellquote($primary_target.peadm::certname())}.cert.pem \

From 4c6e497ae0a5d4df7f790c9774cc148ccfb6856e Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Mon, 2 May 2022 15:50:31 -0700
Subject: [PATCH 09/11] Fix PuppetDB restore ordering, add agent run

---
 plans/restore.pp | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/plans/restore.pp b/plans/restore.pp
index cf7a5a98..228555fa 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -198,16 +198,6 @@
       | CMD
   }
 
-  # Use PuppetDB's /pdb/admin/v1/archive API to MERGE previously saved data
-  # into the restored database.
-  # TODO: consider adding a heuristic to skip when innappropriate due to size
-  #       or other factors.
-  if getvar('recovery_opts.puppetdb') {
-    run_command(@("CMD"/L), $primary_target)
-      /opt/puppetlabs/bin/puppet-db import ${shellquote($recovery_directory)}/puppetdb-archive.bin
-      | CMD
-  }
-
   # Use `puppet infra` to ensure correct file permissions, restart services,
   # etc. Make sure not to try and get config data from the classifier, which
   # isn't yet up and running.
@@ -220,6 +210,19 @@
     /opt/puppetlabs/bin/puppet-infra reinitialize replica -y
     | CMD
 
+  # Use PuppetDB's /pdb/admin/v1/archive API to MERGE previously saved data
+  # into the restored database.
+  # TODO: consider adding a heuristic to skip when innappropriate due to size
+  #       or other factors.
+  if getvar('recovery_opts.puppetdb') {
+    run_command(@("CMD"/L), $primary_target)
+      /opt/puppetlabs/bin/puppet-db import ${shellquote($recovery_directory)}/puppetdb-archive.bin
+      | CMD
+  }
+
+  # Run Puppet to pick up last remaining config tweaks
+  run_task('peadm::puppet_runonce', $primary_target)
+
   apply($primary_target){
     file { $recovery_directory :
       ensure => 'absent',

From b43c4513bcbc9e84359a5a77c4d2bcf0380085f8 Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Fri, 6 May 2022 07:35:01 -0700
Subject: [PATCH 10/11] Fix rubycop style issues

---
 tasks/get_peadm_config.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tasks/get_peadm_config.rb b/tasks/get_peadm_config.rb
index 4193d0ad..52f78057 100755
--- a/tasks/get_peadm_config.rb
+++ b/tasks/get_peadm_config.rb
@@ -24,8 +24,8 @@ def config
     replica_letter = primary_letter.eql?('A') ? 'B' : 'A'
 
     configured_postgresql_servers = [
-      groups.dig("PE Primary A", 'config_data', 'puppet_enterprise::profile::puppetdb', 'database_host'),
-      groups.dig("PE Primary B", 'config_data', 'puppet_enterprise::profile::puppetdb', 'database_host'),
+      groups.dig('PE Primary A', 'config_data', 'puppet_enterprise::profile::puppetdb', 'database_host'),
+      groups.dig('PE Primary B', 'config_data', 'puppet_enterprise::profile::puppetdb', 'database_host'),
     ].compact
 
     postgresql = {

From c7888970b27e596bc86a46b212a5c072540cb6ce Mon Sep 17 00:00:00 2001
From: Reid Vandewiele <reid@puppet.com>
Date: Fri, 6 May 2022 07:52:33 -0700
Subject: [PATCH 11/11] Mark backup and restore plans api-private

They aren't ready yet for full use
---
 plans/backup.pp  | 1 +
 plans/restore.pp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/plans/backup.pp b/plans/backup.pp
index fa30add4..6672f3dd 100644
--- a/plans/backup.pp
+++ b/plans/backup.pp
@@ -1,3 +1,4 @@
+# @api private
 # @summary Backup the core user settings for puppet infrastructure
 #
 # This plan can backup data as outlined at insert doc
diff --git a/plans/restore.pp b/plans/restore.pp
index 228555fa..68a7cd9b 100644
--- a/plans/restore.pp
+++ b/plans/restore.pp
@@ -1,3 +1,4 @@
+# @api private
 # @summary Restore the core user settings for puppet infrastructure from backup
 #
 # This plan can restore data to puppet infrastructure for DR and rebuilds