Merge pull request #23 from johnduarte/slv-366-make-pdb-pg-optional

SLV-366 Make external database host optional

Author: reidmv

documentation/images/PE_Large_Architecture_no_HA.png

@@ -0,0 +1,67 @@
+# PE Large Architecture
+
+
+## Overview
+
+This module can also be used to deploy a Puppet Enterprise Large Architecture.
+Such an deployment differs from an Extra Large Architecture in that it does
+**not** include an external database.  PuppetDB is served from the master.
+
+**NOTE:** Currently, the module does not deploy a Large Architecture with HA.
+The currently supported deployment architecture is shown below.
+
+![Large Architecture without HA](images/PE_Large_Architecture_no_HA.png)
+
+
+## Instructions
+
+The process for deploying a PE Large Architecture is very similar to the
+[basic_usage](basic_usage.md) for deploying the XL Architecture.  These two
+differ only in the parameters supplied to the bolt plans.  Specifically, the
+`puppetdb_database_host`, `master_replica_host`, and
+`puppetdb_database_replica_host` parameters need to be omitted in order to
+deploy a PE Large Architecture.
+
+Ensuring that the parameters above are omitted from the `params.json` file,
+the [basic usage instructions](basic_usage.md#basic-usage-instructions) can be
+used to run the `pe_xl` plan in order to install and configure the deployment.
+
+Example nodes.yaml Bolt inventory file:
+
+```yaml
+---
+groups:
+  - name: pe_xl_nodes
+    config:
+      transport: ssh
+      ssh:
+        host-key-check: false
+        user: centos
+        run-as: root
+        tty: true
+    nodes:
+      - pe-xl-core-0.lab1.puppet.vm
+      - pe-xl-compiler-0.lab1.puppet.vm
+      - pe-xl-compiler-1.lab1.puppet.vm
+```
+
+Example params.json Bolt parameters file:
+
+```json
+{
+  "install": true,
+  "configure": true,
+  "upgrade": false,
+
+  "master_host": "pe-xl-core-0.lab1.puppet.vm",
+  "compiler_hosts": [
+    "pe-xl-compiler-0.lab1.puppet.vm",
+    "pe-xl-compiler-1.lab1.puppet.vm"
+  ],
+
+  "console_password": "puppetlabs",
+  "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ],
+  "compiler_pool_address": "puppet.lab1.puppet.vm",
+  "version": "2018.1.4"
+}
+```

documentation/large_deploy.md

@@ -2,9 +2,9 @@
 #
 plan pe_xl::configure (
   String[1]           $master_host,
-  String[1]           $puppetdb_database_host,
   Array[String[1]]    $compiler_hosts = [ ],
 
+  Optional[String[1]] $puppetdb_database_host = undef,
   Optional[String[1]] $master_replica_host = undef,
   Optional[String[1]] $puppetdb_database_replica_host = undef,
 
@@ -34,12 +34,24 @@
     default => fail('Must specify either both or neither of master_replica_host, puppetdb_database_replica_host'),
   }
 
+  # Ensure primary external database host for HA
+  if $ha {
+    if ! $puppetdb_database_host {
+      fail("Must specify puppetdb_database_host for HA environment")
+    }
+  }
+
   # Allow for the configure task to be run local to the master.
   $master_target = $executing_on_master ? {
     true  => "local://${master_host}",
     false => $master_host,
   }
 
+  $puppetdb_database_target = $puppetdb_database_host ? {
+    undef   => $master_host,
+    default => $puppetdb_database_host,
+  }
+
   # Retrieve and deploy Puppet modules from the Forge so that they can be used
   # for ensuring some configuration (node groups)
   [ ['WhatsARanjit-node_manager', '0.7.1'],
@@ -58,7 +70,7 @@
   run_task('pe_xl::configure_node_groups', $master_target,
     master_host                    => $master_host,
     master_replica_host            => $master_replica_host,
-    puppetdb_database_host         => $puppetdb_database_host,
+    puppetdb_database_host         => $puppetdb_database_target,
     puppetdb_database_replica_host => $puppetdb_database_replica_host,
     compiler_pool_address          => $compiler_pool_address,
   )
@@ -73,7 +85,7 @@
   # Run Puppet on the PuppetDB Database hosts to update their auth
   # configuration to allow the compilers to connect
   run_task('pe_xl::puppet_runonce', [
-    $puppetdb_database_host,
+    $puppetdb_database_target,
     $puppetdb_database_replica_host,
   ].pe_xl::flatten_compact)
 
@@ -100,7 +112,7 @@
   # Run Puppet everywhere to pick up last remaining config tweaks
   run_task('pe_xl::puppet_runonce', [
     $master_target,
-    $puppetdb_database_host,
+    $puppetdb_database_target,
     $compiler_hosts,
     $master_replica_host,
     $puppetdb_database_replica_host,

plans/configure.pp

@@ -2,9 +2,9 @@
 #
 plan pe_xl::install (
   String[1]           $master_host,
-  String[1]           $puppetdb_database_host,
   Array[String[1]]    $compiler_hosts = [ ],
 
+  Optional[String[1]] $puppetdb_database_host = undef,
   Optional[String[1]] $master_replica_host = undef,
   Optional[String[1]] $puppetdb_database_replica_host = undef,
 
@@ -31,7 +31,11 @@
     $master_replica_host,
   ].pe_xl::flatten_compact()
 
-  $ha_database_target = [
+  $puppetdb_database_target = [
+    $puppetdb_database_host,
+  ].pe_xl::flatten_compact()
+
+  $puppetdb_database_replica_target = [
     $puppetdb_database_replica_host,
   ].pe_xl::flatten_compact()
 
@@ -42,6 +46,13 @@
     default => fail('Must specify either both or neither of master_replica_host, puppetdb_database_replica_host'),
   }
 
+  # Ensure primary external database host for HA
+  if $ha {
+    if ! $puppetdb_database_host {
+      fail("Must specify puppetdb_database_host for HA environment")
+    }
+  }
+
   $all_hosts = [
     $core_hosts,
     $ha_hosts,
@@ -111,8 +122,8 @@
 
   # Upload the pe.conf files to the hosts that need them
   pe_xl::file_content_upload($master_pe_conf, '/tmp/pe.conf', $master_host)
-  pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_host)
-  pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $ha_database_target)
+  pe_xl::file_content_upload($puppetdb_database_pe_conf, '/tmp/pe.conf', $puppetdb_database_target)
+  pe_xl::file_content_upload($puppetdb_database_replica_pe_conf, '/tmp/pe.conf', $puppetdb_database_replica_target)
 
   # Download the PE tarball and send it to the nodes that need it
   $pe_tarball_name     = "puppet-enterprise-${version}-el-7-x86_64.tar.gz"
@@ -138,7 +149,7 @@
       | HEREDOC
   )
 
-  run_task('pe_xl::mkdir_p_file', $puppetdb_database_host,
+  run_task('pe_xl::mkdir_p_file', $puppetdb_database_target,
     path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
     content => @("HEREDOC"),
       ---
@@ -149,7 +160,7 @@
       | HEREDOC
   )
 
-  run_task('pe_xl::mkdir_p_file', $ha_database_target,
+  run_task('pe_xl::mkdir_p_file', $puppetdb_database_replica_target,
     path    => '/etc/puppetlabs/puppet/csr_attributes.yaml',
     content => @("HEREDOC"),
       ---
@@ -161,25 +172,31 @@
   )
 
   # Get the master installation up and running. The installer will
-  # "fail" because PuppetDB can't start. That's expected.
+  # "fail" because PuppetDB can't start, if puppetdb_database_host
+  # is set. That's expected.
+  $shortcircuit_puppetdb = $puppetdb_database_host ? {
+    undef   => false,
+    default => true,
+  }
   without_default_logging() || {
     out::message("Starting: task pe_xl::pe_install on ${master_host}")
     run_task('pe_xl::pe_install', $master_host,
-      _catch_errors         => true,
+      _catch_errors         => $shortcircuit_puppetdb,
       tarball               => $upload_tarball_path,
       peconf                => '/tmp/pe.conf',
-      shortcircuit_puppetdb => true,
+      shortcircuit_puppetdb => $shortcircuit_puppetdb,
     )
     out::message("Finished: task pe_xl::pe_install on ${master_host}")
   }
 
   # Configure autosigning for the puppetdb database hosts 'cause they need it
+  $autosign_conf = $database_hosts.reduce |$memo,$host| { "${host}\n${memo}" }
   run_task('pe_xl::mkdir_p_file', $master_host,
     path    => '/etc/puppetlabs/puppet/autosign.conf',
     owner   => 'pe-puppet',
     group   => 'pe-puppet',
     mode    => '0644',
-    content => $database_hosts.reduce |$memo,$host| { "${host}\n${memo}" },
+    content => "$autosign_conf",
   )
 
   # Run the PE installer on the puppetdb database hosts

plans/install.pp

@@ -5,14 +5,14 @@
       "type": "String[1]",
       "description": "The certname of the master"
     },
-    "puppetdb_database_host": {
-      "type": "String[1]",
-      "description": "The certname of the PuppetDB database"
-    },
     "compiler_pool_address": {
       "type": "String[1]",
       "description": "The service name to use for the compiler pool"
     },
+    "puppetdb_database_host": {
+      "type": "Optional[String[1]]",
+      "description": "The certname of the PuppetDB database"
+    },
     "master_replica_host": {
       "type": "Optional[String[1]]",
       "description": "The certname of the master replica"

tasks/configure_node_groups.json

@@ -60,18 +60,21 @@ class configure_node_groups (
     variables => { 'pe_master' => true },
   }
 
-  # This class has to be included here because puppet_enterprise is declared
-  # in the console with parameters. It is therefore not possible to include
-  # puppet_enterprise::profile::database in code without causing a conflict.
-  node_group { 'PE Database':
-    ensure               => present,
-    parent               => 'PE Infrastructure',
-    environment          => 'production',
-    override_environment => false,
-    rule                 => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']],
-    classes              => {
-      'puppet_enterprise::profile::database' => { },
-    },
+  # Create the database group if a database host is external
+  if ($puppetdb_database_host != $master_host) {
+    # This class has to be included here because puppet_enterprise is declared
+    # in the console with parameters. It is therefore not possible to include
+    # puppet_enterprise::profile::database in code without causing a conflict.
+    node_group { 'PE Database':
+      ensure               => present,
+      parent               => 'PE Infrastructure',
+      environment          => 'production',
+      override_environment => false,
+      rule                 => ['and', ['=', ['trusted', 'extensions', 'pp_role'], 'pe_xl::puppetdb_database']],
+      classes              => {
+        'puppet_enterprise::profile::database' => { },
+      },
+    }
   }
 
   # Create data-only groups to store PuppetDB PostgreSQL database configuration

tasks/configure_node_groups.sh

@@ -1,8 +1,8 @@
-<%- | String[1] $console_password,
-      String[1] $master_host,
-      String[1] $puppetdb_database_host,
-      Array     $dns_alt_names,
-      Hash      $r10k_sources,
+<%- | String[1]        $console_password,
+      String[1]        $master_host,
+      Optional[String] $puppetdb_database_host,
+      Array            $dns_alt_names,
+      Hash             $r10k_sources,
 | -%>
 #----------------------------------------------------------------------------
 # Puppet Enterprise installer configuration file
@@ -20,11 +20,13 @@
   "puppet_enterprise::puppet_master_host": "<%= $master_host %>"
   "pe_install::puppet_master_dnsaltnames": <%= $dns_alt_names %>
 
+  <% if $puppetdb_database_host =~ String[1] { -%>
   # PuppetDB Database configuration
   # This parameter does not change the "role" of the master; the
   # master still considers itself a database host. It will reconfigure
   # PuppetDB though to use a different PostgreSQL host for that database.
   "puppet_enterprise::profile::puppetdb::database_host": "<%= $puppetdb_database_host %>"
+  <% } -%>
 
   # Code Manager
   # An initial configuration for code manager is required in order to fully