diff --git a/metadata.json b/metadata.json index 3ae89dda..d82a68a8 100644 --- a/metadata.json +++ b/metadata.json @@ -31,6 +31,10 @@ { "name": "puppetlabs/service", "version_requirement": ">= 1.3.0 < 3.0.0" + }, + { + "name": "puppetlabs/inifile", + "version_requirement": ">= 5.2.0 < 6.0.0" } ], "operatingsystem_support": [ diff --git a/plans/add_database.pp b/plans/add_database.pp new file mode 100644 index 00000000..bda848dd --- /dev/null +++ b/plans/add_database.pp @@ -0,0 +1,230 @@ +plan peadm::add_database( + Peadm::SingleTargetSpec $targets, + Peadm::SingleTargetSpec $primary_host, + Optional[Enum['init', 'pair']] $mode = undef, + Optional[Enum[ + 'init-db-node', + 'replicate-db', + 'update-classification', + 'update-db-settings', + 'cleanup-db', + 'finalize']] $begin_at_step = undef, +) { + + $primary_target = peadm::get_targets($primary_host, 1) + + # Get current peadm config before making modifications and shutting down + # PuppetDB + $peadm_config = run_task('peadm::get_peadm_config', $primary_target).first.value + + $compilers = $peadm_config['params']['compilers'] + + # Bail if this is trying to be ran against Standard + if $compilers.empty { + fail_plan('Plan Peadm::Add_database only applicable for L and XL deployments') + } + + # Existing nodes and their assignments + $replica_host = $peadm_config['params']['replica_host'] + $primary_postgresql_host = $peadm_config['params']['primary_postgresql_host'] + $replica_postgresql_host = $peadm_config['params']['replica_postgresql_host'] + + $replica_target = peadm::get_targets($replica_host, 1) + + # Pluck these out for determining group letter assignments + $roles = $peadm_config['role-letter'] + + # Override mode in case of failure on previous run + if $mode { + $operating_mode = $mode + out::message("Operating mode overridden by parameter mode set to ${mode}") + } else { + # If array is empty then no external databases were previously configured + $no_external_db = peadm::flatten_compact([ + $primary_postgresql_host, + $replica_postgresql_host + ]).empty + + # Pick operating mode based on array check + if $no_external_db { + $operating_mode = 'init' + } else { + $operating_mode = 'pair' + } + } + out::message("Operating in ${operating_mode} mode") + + if $operating_mode == 'init' { + # If no other PSQL node then match primary group letter + $avail_group_letter = peadm::flatten_compact($roles['server'].map |$k,$v| { + if $v == $primary_host { + $k + } + })[0] + # Assume PuppetDB backend hosted on Primary if in init mode + $source_db_host = $primary_host + } else { + # The letter which doesn't yet have a server assigned or in the event this + # is a replacement operation, the letter this node was assigned to previously + $avail_group_letter = peadm::flatten_compact($roles['postgresql'].map |$k,$v| { + if (! $v) or ($v == $targets.peadm::certname()) { + $k + } + })[0] + # When in pair mode we assume the other PSQL node will serve as our source + $source_db_host = peadm::flatten_compact([ + $primary_postgresql_host, + $replica_postgresql_host + ]).reject($targets.peadm::certname())[0] + } + + out::message("Adding PostgreSQL server ${targets.peadm::certname()} to availability group ${avail_group_letter}") + out::message("Using ${source_db_host} to populate ${targets.peadm::certname()}") + + $source_db_target = peadm::get_targets($source_db_host, 1) + + peadm::plan_step('init-db-node') || { + # Install PSQL on new node to be used as external PuppetDB backend by using + # puppet in lieu of installer + run_plan('peadm::subplans::component_install', $targets, + primary_host => $primary_target, + avail_group_letter => $avail_group_letter, + role => 'puppet/puppetdb-database' + ) + } + + # Stop Puppet to ensure catalogs are not being compiled for PE infrastructure nodes + run_command('systemctl stop puppet.service', peadm::flatten_compact([ + $targets, + $compilers, + $primary_target, + $replica_target, + $source_db_target + ])) + + # Stop frontend compiler services that causes changes to PuppetDB backend when + # agents request catalogs + run_command('systemctl stop pe-puppetserver.service pe-puppetdb.service', $compilers) + + peadm::plan_step('replicate-db') || { + # Replicate content from source to newly installed PSQL server + run_plan('peadm::subplans::db_populate', $targets, source_host => $source_db_target.peadm::certname()) + + # Run Puppet on new PSQL node to fix up certificates and permissions + run_task('peadm::puppet_runonce', $targets) + } + + if $operating_mode == 'init' { + + # Update classification and database.ini settings, assume a replica PSQL + # does not exist + peadm::plan_step('update-classification') || { + run_plan('peadm::util::update_classification', $primary_target, + primary_postgresql_host => pick($primary_postgresql_host, $targets), + peadm_config => $peadm_config + ) + } + + peadm::plan_step('update-db-settings') || { + run_plan('peadm::util::update_db_setting', peadm::flatten_compact([ + $compilers, + $primary_target, + $replica_target + ]), + primary_postgresql_host => $targets, + peadm_config => $peadm_config + ) + + # (Re-)Start PuppetDB now that we are done making modifications + run_command('systemctl restart pe-puppetdb.service', peadm::flatten_compact([ + $primary_target, + $replica_target + ])) + } + + # Clean up old puppetdb database on primary and those which were copied to + # new host. + peadm::plan_step('cleanup-db') || { + + $target_db_purge = [ + 'pe-activity', + 'pe-classifier', + 'pe-inventory', + 'pe-orchestrator', + 'pe-rbac' + ] + + # If a primary replica exists then pglogical is enabled and will prevent + # the clean up of databases on our target because it opens a connection. + if $replica_host { + run_plan('peadm::util::db_disable_pglogical', $targets, databases => $target_db_purge) + } + + # Clean up old databases + $clean_source = peadm::flatten_compact([ + $source_db_target, + $primary_target, + $replica_target + ]) + + run_plan('peadm::util::db_purge', $clean_source, databases => ['pe-puppetdb']) + run_plan('peadm::util::db_purge', $targets, databases => $target_db_purge) + } + } else { + peadm::plan_step('update-classification') || { + run_plan('peadm::util::update_classification', $primary_target, + primary_postgresql_host => pick($primary_postgresql_host, $targets), + replica_postgresql_host => pick($replica_postgresql_host, $targets), + peadm_config => $peadm_config + ) + } + + # Plan needs to know which node is being added as well as primary and + # replica designation + peadm::plan_step('update-db-settings') || { + run_plan('peadm::util::update_db_setting', peadm::flatten_compact([ + $compilers, + $primary_target, + $replica_target + ]), + new_postgresql_host => $targets, + primary_postgresql_host => pick($primary_postgresql_host, $targets), + replica_postgresql_host => pick($replica_postgresql_host, $targets), + peadm_config => $peadm_config + ) + + # (Re-)Start PuppetDB now that we are done making modifications + run_command('systemctl restart pe-puppetdb.service', peadm::flatten_compact([ + $primary_target, + $replica_target + ])) + } + peadm::plan_step('cleanup-db') || { + out::message("No databases to cleanup when in ${operating_mode}") + } + } + + # Start frontend compiler services so catalogs can once again be compiled by + # agents + run_command('systemctl start pe-puppetserver.service pe-puppetdb.service', $compilers) + + + peadm::plan_step('finalize') || { + # Run Puppet to sweep up but no restarts should occur so do them in parallel + run_task('peadm::puppet_runonce', peadm::flatten_compact([ + $targets, + $primary_target, + $compilers, + $replica_target + ])) + + # Start Puppet agent + run_command('systemctl start puppet.service', peadm::flatten_compact([ + $targets, + $compilers, + $primary_target, + $replica_target, + $source_db_target + ])) + } +} diff --git a/plans/subplans/component_install.pp b/plans/subplans/component_install.pp new file mode 100644 index 00000000..29159673 --- /dev/null +++ b/plans/subplans/component_install.pp @@ -0,0 +1,32 @@ +# @api private +# +# @summary Install a new PEADM component +# @param avail_group_letter _ Either A or B; whichever of the two letter designations the component is assigned to +# @param dns_alt_names _ A comma_separated list of DNS alt names for the component +# @param database_host _ The hostname and certname of the new component +# @param primary_host _ The hostname the primary Puppet server +# @param role _ Optional PEADM role the component will serve +plan peadm::subplans::component_install( + Peadm::SingleTargetSpec $targets, + Peadm::SingleTargetSpec $primary_host, + Enum['A', 'B'] $avail_group_letter, + Optional[String[1]] $dns_alt_names = undef, + Optional[String[1]] $role = undef +){ + $component_target = peadm::get_targets($targets, 1) + $primary_target = peadm::get_targets($primary_host, 1) + + run_plan('peadm::subplans::prepare_agent', $component_target, + primary_host => $primary_target, + dns_alt_name => $dns_alt_names, + certificate_extensions => { + peadm::oid('peadm_role') => $role, + peadm::oid('peadm_availability_group') => $avail_group_letter, + } + ) + + # On component, run the puppet agent to finish initial configuring of component + run_task('peadm::puppet_runonce', $component_target) + + return("Installation of component ${$component_target.peadm::certname()} with peadm_role: ${role} succeeded.") +} diff --git a/plans/subplans/db_populate.pp b/plans/subplans/db_populate.pp new file mode 100644 index 00000000..fac0cff6 --- /dev/null +++ b/plans/subplans/db_populate.pp @@ -0,0 +1,95 @@ +# This plan is in development and currently considered experimental. +# +# @api private +# +# @summary Destructively (re)populates a new or existing database with the contents or a known good source +# @param source_host _ The hostname of the database containing data +plan peadm::subplans::db_populate( + Peadm::SingleTargetSpec $targets, + Peadm::SingleTargetSpec $source_host, +) { + $source_target = peadm::get_targets($source_host, 1) + $destination_target = peadm::get_targets($targets, 1) + + # Always ensure Puppet is stopped or it'll remove rules that allow replication + run_command('systemctl stop puppet.service', peadm::flatten_compact([ + $source_target, + $destination_target, + ])) + + # Add the following two lines to /opt/puppetlabs/server/data/postgresql/11/data/pg_ident.conf + # + # These lines allow connections from destination by pg_basebackup to replicate + # content + apply($source_target) { + file_line { 'replication-pe-ha-replication-map': + path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_ident.conf', + line => "replication-pe-ha-replication-map ${destination_target.peadm::certname()} pe-ha-replication", + } + file_line { 'replication-pe-ha-replication-ipv4': + path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_hba.conf', + line => 'hostssl replication pe-ha-replication 0.0.0.0/0 cert map=replication-pe-ha-replication-map clientcert=1', + } + file_line { 'replication-pe-ha-replication-ipv6': + path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_hba.conf', + line => 'hostssl replication pe-ha-replication ::/0 cert map=replication-pe-ha-replication-map clientcert=1', + } + } + + # Reload pe-postgresql to activate replication rules + run_command('systemctl reload pe-postgresql.service', $source_target) + + # Save existing certificates to use for authentication to source. Can not use + # certs stored in /etc/puppetlabs/puppet/ssl because we will run pg_basebackup + # as pe-postgres user, which lacks access + run_command('mv /opt/puppetlabs/server/data/postgresql/11/data/certs /opt/puppetlabs/server/data/pg_certs', $destination_target) + + # pg_basebackup requires an entirely empty data directory + run_command('rm -rf /opt/puppetlabs/server/data/postgresql/*', $destination_target) + + $pg_basebackup = @("PGBASE") + runuser -u pe-postgres -- \ + /opt/puppetlabs/server/bin/pg_basebackup \ + -D /opt/puppetlabs/server/data/postgresql/11/data \ + -d "host=${source_host} + user=pe-ha-replication + sslmode=verify-full + sslcert=/opt/puppetlabs/server/data/pg_certs/_local.cert.pem + sslkey=/opt/puppetlabs/server/data/pg_certs/_local.private_key.pem + sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem" + | - PGBASE + + run_command($pg_basebackup, $destination_target) + + # Delete the saved certs, they'll be properly re-populated by an agent run + run_command('rm -rf /opt/puppetlabs/server/data/pg_certs', $destination_target) + + # Start pe-postgresql.service + run_command('systemctl start pe-postgresql.service', $destination_target) + + # Delete the previously add replication rules to prevent Puppet restarting + # thing later + apply($source_target) { + file_line { 'replication-pe-ha-replication-map': + ensure => absent, + path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_ident.conf', + line => "replication-pe-ha-replication-map ${destination_target.peadm::certname()} pe-ha-replication", + } + file_line { 'replication-pe-ha-replication-ipv4': + ensure => absent, + path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_hba.conf', + line => 'hostssl replication pe-ha-replication 0.0.0.0/0 cert map=replication-pe-ha-replication-map clientcert=1', + } + file_line { 'replication-pe-ha-replication-ipv6': + ensure => absent, + path => '/opt/puppetlabs/server/data/postgresql/11/data/pg_hba.conf', + line => 'hostssl replication pe-ha-replication ::/0 cert map=replication-pe-ha-replication-map clientcert=1', + } + } + + # Reload pe-postgresql to revoke replication rules + run_command('systemctl reload pe-postgresql.service', $source_target) + + return("Population of ${$destination_target.peadm::certname()} with data from s${$source_target.peadm::certname()} succeeded.") + +} diff --git a/plans/subplans/prepare_agent.pp b/plans/subplans/prepare_agent.pp new file mode 100644 index 00000000..8f4f0508 --- /dev/null +++ b/plans/subplans/prepare_agent.pp @@ -0,0 +1,48 @@ +plan peadm::subplans::prepare_agent ( + Peadm::SingleTargetSpec $targets, + Peadm::SingleTargetSpec $primary_host, + Hash $certificate_extensions, + Optional[Array] $dns_alt_names = undef, +) { + + $agent_target = peadm::get_targets($targets, 1) + $primary_target = peadm::get_targets($primary_host, 1) + + $dns_alt_names_flag = $dns_alt_names? { + undef => [], + default => ["main:dns_alt_names=${dns_alt_names}"], + } + + $status = run_task('package', $agent_target, + action => 'status', + name => 'puppet-agent').first['status'] + + if $status == 'uninstalled' { + run_plan('peadm::util::insert_csr_extension_requests', $agent_target, + extension_requests => $certificate_extensions + ) + run_task('peadm::agent_install', $agent_target, + server => $primary_target.peadm::certname(), + install_flags => $dns_alt_names_flag + [ + '--puppet-service-ensure', 'stopped', + "main:certname=${agent_target.peadm::certname()}", + ], + ) + } + + # Ensures scenarios where agent was pre-installed but never on-boarding and + # when agent was absent but their was an existing signed certificate with the + # same name as the one being provisioned. + # + # If necessary, manually submit a CSR + # ignoring errors to simplify logic + run_task('peadm::submit_csr', $agent_target, {'_catch_errors' => true}) + + # On primary, if necessary, sign the certificate request + run_task('peadm::sign_csr', $primary_target, { 'certnames' => [$agent_target.peadm::certname()] } ) + + run_plan('peadm::modify_certificate', $agent_target, + primary_host => $primary_target.peadm::certname(), + add_extensions => $certificate_extensions + ) +} diff --git a/plans/util/db_disable_pglogical.pp b/plans/util/db_disable_pglogical.pp new file mode 100644 index 00000000..b2fd23ed --- /dev/null +++ b/plans/util/db_disable_pglogical.pp @@ -0,0 +1,17 @@ +plan peadm::util::db_disable_pglogical( + Peadm::SingleTargetSpec $targets, + Array[String[1]] $databases, +) { + + # Simplest way to disable the connection that the pglogical supervisor opens + # to each database it means to replicate. + $databases.each |$database| { + run_command( "runuser -u pe-postgres -- \ + /opt/puppetlabs/server/bin/psql \"${database}\" -c 'DROP EXTENSION IF EXISTS pglogical'", + $targets + ) + } + + # Reload does not work to shutdown the connection post extension removal, must restart + run_command('systemctl restart pe-postgresql', $targets) +} diff --git a/plans/util/db_purge.pp b/plans/util/db_purge.pp new file mode 100644 index 00000000..18e046f4 --- /dev/null +++ b/plans/util/db_purge.pp @@ -0,0 +1,19 @@ +plan peadm::util::db_purge( + TargetSpec $targets, + Array[String[1]] $databases, +) { + + # Their are more sophisticated ways to clean up these databases so they do not + # continue taking up disk space but they are finicky and grow complex. Instead + # just delete them even though Puppet will recreate them on the next agent run. + $databases.each |$database| { + run_command( "runuser -u pe-postgres -- \ + /opt/puppetlabs/server/bin/psql pe-postgres -c 'DROP DATABASE IF EXISTS \"${database}\"'", + $targets + ) + run_command("runuser -u pe-postgres -- \ + /opt/puppetlabs/server/bin/psql pe-postgres -c 'DROP TABLESPACE IF EXISTS \"${database}\"'", + $targets + ) + } +} diff --git a/plans/util/update_classification.pp b/plans/util/update_classification.pp new file mode 100644 index 00000000..dff731f9 --- /dev/null +++ b/plans/util/update_classification.pp @@ -0,0 +1,76 @@ +# @api private +# +# @summary Configure classification +# +plan peadm::util::update_classification ( + # Standard + Peadm::SingleTargetSpec $targets, + Optional[Hash] $peadm_config = undef, + Optional[Peadm::SingleTargetSpec] $replica_host = undef, + + # Extra Large + Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, + Optional[Peadm::SingleTargetSpec] $replica_postgresql_host = undef, + + # Common Configuration + Optional[String] $compiler_pool_address = undef, + Optional[String] $internal_compiler_a_pool_address = undef, + Optional[String] $internal_compiler_b_pool_address = undef, +) { + + # Convert inputs into targets. + $primary_target = peadm::get_targets($targets, 1) + $replica_target = peadm::get_targets($replica_host, 1) + $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) + + # Makes this more easily usable outside a plan + if $peadm_config { + $current = $peadm_config['params'] + } else { + $current = run_task('peadm::get_peadm_config', $primary_target).first.value['params'] + } + + # When a replica in configured, the B side of the deployment requires that + # replica_postgresql_host to be set, if it is not then PuppetDB will be left + # non-functional. Doing this will allow both sides of the deployment to start + # up and be functional until the second PostgreSQL node can be provisioned and configured. + if (! $replica_postgresql_target.peadm::certname()) and $current['replica_host'] { + out::message('Overriding replica_postgresql_host while in transitive state') + $overridden_replica_postgresql_target = $primary_postgresql_target + } else { + $overridden_replica_postgresql_target = $replica_postgresql_target + } + + $new = merge($current, { + 'primary_host' => $primary_target.peadm::certname(), + 'replica_host' => $replica_target.peadm::certname(), + 'primary_postgresql_host' => $primary_postgresql_target.peadm::certname(), + 'replica_postgresql_host' => $overridden_replica_postgresql_target.peadm::certname(), + 'compiler_pool_address' => $compiler_pool_address, + 'internal_compiler_a_pool_address' => $internal_compiler_a_pool_address, + 'internal_compiler_b_pool_address' => $internal_compiler_b_pool_address + }) + + out::message('Classification to be updated using the following hash...') + out::message($new) + + apply($primary_target) { + class { 'peadm::setup::node_manager_yaml': + primary_host => $primary_target.peadm::certname(), + } + + class { 'peadm::setup::node_manager': + primary_host => $new['primary_host'], + server_a_host => $new['primary_host'], + server_b_host => $new['replica_host'], + postgresql_a_host => $new['primary_postgresql_host'], + postgresql_b_host => $new['replica_postgresql_host'], + compiler_pool_address => $new['compiler_pool_address'], + internal_compiler_a_pool_address => $new['internal_compiler_a_pool_address'], + internal_compiler_b_pool_address => $new['internal_compiler_b_pool_address'], + require => Class['peadm::setup::node_manager_yaml'], + } + } + return('The classification of Puppet Enterprise components has succeeded.') +} diff --git a/plans/util/update_db_setting.pp b/plans/util/update_db_setting.pp new file mode 100644 index 00000000..909581c5 --- /dev/null +++ b/plans/util/update_db_setting.pp @@ -0,0 +1,75 @@ +# @api private +# +# @summary Make updates to PuppetDB database settings +# +plan peadm::util::update_db_setting ( + TargetSpec $targets, + Optional[Peadm::SingleTargetSpec] $new_postgresql_host = undef, + Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, + Optional[Peadm::SingleTargetSpec] $replica_postgresql_host = undef, + Optional[Hash] $peadm_config = undef, +) { + + # Convert inputs into targets. + $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) + + # Originally written to handle some additional logic which was eventually + # determined to not be useful and was pulled out. As a result could use + # more additional simplification. The goal is to match each infrastructure + # component to the PostgreSQL nodes which corresponds to their availability + # letter and if a match is not found, assume that new node is the match. + # + # FIX ME: Test removal of $primary_potsgresql_host and $replica_postgresql_host + # parameter check. Likely only parameter needed is the node be added. Section + # also needs to be parallelized, can't use built functionality of apply(). + get_targets($targets).each |$target| { + + # Availability group does not matter if only one PSQL node in the cluster + if ($primary_postgresql_host and $replica_postgresql_host) { + + # Existing config used to dynamically pair nodes with appropriate PSQL + # server + $roles = $peadm_config['role-letter'] + + # Determine configuration by pairing target with existing availability letter + # assignments, setting to the new node if no match is found. + $target_group_letter = peadm::flatten_compact([$roles['compilers'],$roles['server']].map |$role| { + $role.map |$k,$v| { + if $target.peadm::certname() in $v { $k } + } + })[0] + $match = $roles['postgresql'][$target_group_letter] + if $match { + $db = $match + } else { + $db = $new_postgresql_host + } + + $db_setting = "//${db}:5432/pe-puppetdb?ssl=true&sslfactory=org.postgresql.ssl.jdbc4.LibPQFactory&sslmode=verify-full&sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem&sslkey=/etc/puppetlabs/puppetdb/ssl/${target.peadm::certname()}.private_key.pk8&sslcert=/etc/puppetlabs/puppetdb/ssl/${$target.peadm::certname()}.cert.pem" + } else { + $db_setting = "//${primary_postgresql_host}:5432/pe-puppetdb?ssl=true&sslfactory=org.postgresql.ssl.jdbc4.LibPQFactory&sslmode=verify-full&sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem&sslkey=/etc/puppetlabs/puppetdb/ssl/${target.peadm::certname()}.private_key.pk8&sslcert=/etc/puppetlabs/puppetdb/ssl/${$target.peadm::certname()}.cert.pem" + } + + # Introduced new dependency for PEADM to enable modification of INI files + apply($target) { + ini_setting { 'database_setting': + ensure => present, + path => '/etc/puppetlabs/puppetdb/conf.d/database.ini', + section => 'database', + setting => 'subname', + value => $db_setting, + } + + ini_setting { 'read_database_setting': + ensure => present, + path => '/etc/puppetlabs/puppetdb/conf.d/read_database.ini', + section => 'read-database', + setting => 'subname', + value => $db_setting, + } + } + } + + return('PuppetDB database settings were updated successfully.') +}