|
| 1 | +# @summary Restore the core user settings for puppet infrastructure from backup |
| 2 | +# |
| 3 | +# This plan can restore data to puppet infrastructure for DR and rebuilds |
| 4 | +# |
| 5 | +plan peadm::restore ( |
| 6 | + # Standard |
| 7 | + Peadm::SingleTargetSpec $primary_host, |
| 8 | + # Which data to restore |
| 9 | + Boolean $restore_orchestrator = true, |
| 10 | + Boolean $restore_rbac = true, |
| 11 | + Boolean $restore_activity = true, |
| 12 | + Boolean $restore_ca_ssl = true, |
| 13 | + Boolean $restore_puppetdb = false, |
| 14 | + Boolean $restore_classification = true, |
| 15 | + String $input_directory = '/tmp', |
| 16 | + String $working_directory = '/tmp', |
| 17 | + String $backup_timestamp, |
| 18 | +){ |
| 19 | + peadm::assert_supported_bolt_version() |
| 20 | + $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value |
| 21 | + |
| 22 | + $arch = peadm::assert_supported_architecture( |
| 23 | + $primary_host, |
| 24 | + $cluster['params']['replica_host'], |
| 25 | + $cluster['params']['primary_postgresql_host'], |
| 26 | + $cluster['params']['replica_postgresql_host'], |
| 27 | + $cluster['params']['compiler_hosts'], |
| 28 | + ) |
| 29 | + $servers = delete_undef_values([$primary_host , $cluster['params']['replica_host'] ]) |
| 30 | + $cluster_servers = delete_undef_values($servers + $cluster['params']['compiler_hosts'] + [ $cluster['params']['primary_postgresql_host'], $cluster['params']['replica_postgresql_host']]) # lint:ignore:140chars |
| 31 | + if $cluster['params']['compiler_hosts'] { |
| 32 | + $check_puppetdb_on_compilers = run_task('service', $cluster['params']['compiler_hosts'], |
| 33 | + action => 'status', |
| 34 | + name => 'pe-puppetdb' |
| 35 | + ) |
| 36 | + $puppetdb_on_compilers = $check_puppetdb_on_compilers.filter_set | $result | { |
| 37 | + $result['enabled'] == 'enabled' |
| 38 | + }.targets |
| 39 | + } else { |
| 40 | + $puppetdb_on_compilers = undef |
| 41 | + } |
| 42 | + $puppetdb_servers = delete_undef_values([$servers,$puppetdb_on_compilers]) |
| 43 | + $backup_directory = "${input_directory}/pe-backup-${backup_timestamp}" |
| 44 | + $database_backup_directory = "${working_directory}/pe-backup-databases-${backup_timestamp}" |
| 45 | + # I need the actual hostname for the certificate in a remote puppetdb backup. If a user sends primary host as IP it will fail |
| 46 | + $primary_host_fqdn = $cluster['params']['primary_host'] |
| 47 | + $primary_postgresql_host = $cluster['params']['primary_postgresql_host'] |
| 48 | + apply($primary_host){ |
| 49 | + file { $database_backup_directory : |
| 50 | + ensure => 'directory', |
| 51 | + owner => 'pe-puppetdb', |
| 52 | + group => 'pe-postgres', |
| 53 | + mode => '0770' |
| 54 | + } |
| 55 | + } |
| 56 | + |
| 57 | + # Create an array of the names of databases and whether they have to be backed up to use in a lambda later |
| 58 | + $database_to_restore = [ $restore_orchestrator, $restore_activity, $restore_rbac, $restore_puppetdb] |
| 59 | + $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] |
| 60 | + |
| 61 | + peadm::assert_supported_bolt_version() |
| 62 | + |
| 63 | + if $restore_classification { |
| 64 | + |
| 65 | + out::message('# Restoring classification') |
| 66 | + run_task('peadm::backup_classification', $primary_host, |
| 67 | + directory => $working_directory |
| 68 | + ) |
| 69 | + out::message("# Backed up current classification to ${working_directory}/classification_backup.json") |
| 70 | + |
| 71 | + run_task('peadm::transform_classification_groups', $primary_host, |
| 72 | + source_directory => $backup_directory, |
| 73 | + working_directory => $working_directory |
| 74 | + ) |
| 75 | + |
| 76 | + run_task('peadm::restore_classification', $primary_host, |
| 77 | + classification_file => "${working_directory}/classification_backup.json", |
| 78 | + ) |
| 79 | + } |
| 80 | + |
| 81 | + if $restore_ca_ssl { |
| 82 | + out::message('# Restoring ca and ssl certificates') |
| 83 | + run_command("/opt/puppetlabs/bin/puppet-backup restore ${backup_directory}/pe_backup-*tgz --scope=certs --tempdir=${working_directory} --force", $primary_host) # lint:ignore:140chars |
| 84 | + } |
| 85 | + |
| 86 | + ## shutdown services |
| 87 | + run_task('service', $servers, |
| 88 | + action => 'stop', |
| 89 | + name => 'pe-console-services' |
| 90 | + ) |
| 91 | + run_task('service', $primary_host, |
| 92 | + action => 'stop', |
| 93 | + name => 'pe-nginx' |
| 94 | + ) |
| 95 | + run_task('service', $servers, |
| 96 | + action => 'stop', |
| 97 | + name => 'pe-puppetserver' |
| 98 | + ) |
| 99 | + run_task('service', $servers, |
| 100 | + action => 'stop', |
| 101 | + name => 'pxp-agent' |
| 102 | + ) |
| 103 | + run_task('service', $primary_host, |
| 104 | + action => 'stop', |
| 105 | + name => 'pe-orchestration-services' |
| 106 | + ) |
| 107 | + run_task('service', $cluster_servers, |
| 108 | + action => 'stop', |
| 109 | + name => 'puppet' |
| 110 | + ) |
| 111 | + run_task('service', $puppetdb_servers , |
| 112 | + action => 'stop', |
| 113 | + name => 'pe-puppetdb' |
| 114 | + ) |
| 115 | + |
| 116 | + |
| 117 | + # Restore secrets/keys.json if it exists |
| 118 | + out::message('# Restoring ldap secret key if it exists') |
| 119 | + run_command("test -f ${backup_directory}//keys.json && cp -rp ${backup_directory}/keys.json /etc/puppetlabs/console-services/conf.d/secrets/ || echo secret ldap key doesnt exist" , $primary_host) # lint:ignore:140chars |
| 120 | + |
| 121 | + # IF restoring orchestrator restore the secrets too /etc/puppetlabs/orchestration-services/conf.d/secrets/ |
| 122 | + if $restore_orchestrator { |
| 123 | + out::message('# Restoring orchestrator secret keys') |
| 124 | + run_command("cp -rp ${backup_directory}/secrets/* /etc/puppetlabs/orchestration-services/conf.d/secrets ", $primary_host) |
| 125 | + } |
| 126 | + |
| 127 | + $database_to_restore.each |Integer $index, Boolean $value | { |
| 128 | + if $value { |
| 129 | + out::message("# Restoring database ${database_names[$index]}") |
| 130 | + # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary. |
| 131 | + if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host { |
| 132 | + # Drop pglogical extensions and schema if present |
| 133 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"", $primary_postgresql_host) # lint:ignore:140chars |
| 134 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'\"", $primary_postgresql_host) # lint:ignore:140chars |
| 135 | + # To allow pe-puppetdb to restore the database grant temporary privileges |
| 136 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'ALTER USER \\\"pe-puppetdb\\\" WITH SUPERUSER;'\"", $primary_postgresql_host) # lint:ignore:140chars |
| 137 | + # Restore database |
| 138 | + run_command("/opt/puppetlabs/server/bin/pg_restore -d \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host_fqdn}.cert.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host_fqdn}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb user=pe-puppetdb\" -Fd ${backup_directory}/puppetdb_*" , $primary_host) # lint:ignore:140chars |
| 139 | + # Remove pe-puppetdb privileges post restore |
| 140 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'ALTER USER \\\"pe-puppetdb\\\" WITH NOSUPERUSER;'\"", $primary_postgresql_host) # lint:ignore:140chars |
| 141 | + # Drop pglogical extension and schema (again) if present after db restore |
| 142 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"",$primary_postgresql_host) # lint:ignore:140chars |
| 143 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;;'\"",$primary_postgresql_host) # lint:ignore:140chars |
| 144 | + } else { |
| 145 | + # Drop pglogical extensions and schema if present |
| 146 | + run_command("su - pe-postgres -s '/bin/bash' -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"", $primary_host) # lint:ignore:140chars |
| 147 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'\"", $primary_host) # lint:ignore:140chars |
| 148 | + # Restore database |
| 149 | + run_command("cp -pr ${backup_directory}/${database_names[$index]}_* ${database_backup_directory}/ ", $primary_host ) |
| 150 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/pg_restore ${database_backup_directory}/${database_names[$index]}_* -Fd -j4 --dbname=${database_names[$index]}\"", $primary_host)# lint:ignore:140chars |
| 151 | + run_command("sudo -H -u pe-postgres /opt/puppetlabs/server/bin/pg_restore -d ${database_names[$index]} -c ${database_backup_directory}/${database_names[$index]}_*",$primary_host) # lint:ignore:140chars |
| 152 | + run_command("rm -rf ${database_backup_directory}/${database_names[$index]}_*", $primary_host ) |
| 153 | + # Drop pglogical extension and schema (again) if present after db restore |
| 154 | + run_command("su - pe-postgres -s '/bin/bash' -c \"/opt/puppetlabs/server/bin/psql --tuples-only -d '${database_names[$index]}' -c 'DROP SCHEMA IF EXISTS pglogical CASCADE;'\"",$primary_host) # lint:ignore:140chars |
| 155 | + run_command("su - pe-postgres -s /bin/bash -c \"/opt/puppetlabs/server/bin/psql -d '${database_names[$index]}' -c 'DROP EXTENSION IF EXISTS pglogical CASCADE;'\"",$primary_host) # lint:ignore:140chars |
| 156 | + } |
| 157 | + } |
| 158 | + } |
| 159 | + |
| 160 | + ## Restart services |
| 161 | + run_task('service', $primary_host, |
| 162 | + action => 'start', |
| 163 | + name => 'pe-orchestration-services' |
| 164 | + ) |
| 165 | + run_task('service', $servers, |
| 166 | + action => 'start', |
| 167 | + name => 'pxp-agent' |
| 168 | + ) |
| 169 | + run_task('service', $servers, |
| 170 | + action => 'start', |
| 171 | + name => 'pe-puppetserver' |
| 172 | + ) |
| 173 | + run_task('service', $primary_host, |
| 174 | + action => 'start', |
| 175 | + name => 'pe-nginx' |
| 176 | + ) |
| 177 | + run_task('service', $servers, |
| 178 | + action => 'start', |
| 179 | + name => 'pe-console-services' |
| 180 | + ) |
| 181 | + run_task('service', $cluster_servers, |
| 182 | + action => 'start', |
| 183 | + name => 'puppet' |
| 184 | + ) |
| 185 | + run_task('service', $puppetdb_servers, |
| 186 | + action => 'start', |
| 187 | + name => 'pe-puppetdb' |
| 188 | + ) |
| 189 | +# If we have replicas reinitalise any databases restored |
| 190 | + if $cluster['params']['replica_host'] { |
| 191 | + $database_to_restore.each |Integer $index, Boolean $value | { |
| 192 | + if $database_names[$index] != 'pe-puppetdb' and $cluster['params']['replica_postgresql_host'] { |
| 193 | + run_command("/opt/puppetlabs/bin/puppet-infra reinitialize replica --db ${database_names[$index]} -y", $cluster['params']['replica_host'] ) # lint:ignore:140chars |
| 194 | + } |
| 195 | + } |
| 196 | + } |
| 197 | + |
| 198 | + apply($primary_host){ |
| 199 | + file { $database_backup_directory : |
| 200 | + ensure => 'absent', |
| 201 | + force => true |
| 202 | + } |
| 203 | + } |
| 204 | +} |
0 commit comments