From 5e139ba4ee61ef69f8e6805fdbc2681ba9174257 Mon Sep 17 00:00:00 2001 From: Corey Osman Date: Thu, 30 Sep 2021 19:03:33 -0700 Subject: [PATCH 1/4] Remove inotify and log watching service from docker examples * Previously we could spit out logs of what bolt was doing. This feature is now natively supported in bolt and there is no need for installing inotify and supporting services. * Additionally, this adds the old pe installer gpg key * Reformats json and yaml files under spec/docker --- spec/docker/.dockerignore | 1 + spec/docker/Dockerfile | 19 ++++--- .../docker/extra-large-ha/docker-compose.yaml | 12 +++-- spec/docker/extra-large-ha/params.json | 10 +++- .../docker/extra-large-ha/upgrade_params.json | 14 ++--- spec/docker/extra-large/docker-compose.yaml | 4 +- spec/docker/extra-large/params.json | 9 +++- spec/docker/extra-large/upgrade_params.json | 2 +- spec/docker/large-ha/docker-compose.yaml | 5 +- spec/docker/large-ha/params.json | 9 +++- spec/docker/large-ha/upgrade_params.json | 11 ++-- spec/docker/large/docker-compose.yaml | 3 +- spec/docker/large/params.json | 11 ++-- spec/docker/large/upgrade_params.json | 9 ++-- spec/docker/live_audit.service | 8 --- spec/docker/live_audit.sh | 20 ------- spec/docker/pe-old-pub.gpg | 52 +++++++++++++++++++ spec/docker/standard-ha/docker-compose.yaml | 7 ++- spec/docker/standard-ha/params.json | 5 +- spec/docker/standard-ha/upgrade_params.json | 7 ++- spec/docker/standard/docker-compose.yaml | 2 + spec/docker/standard/params.json | 8 +-- spec/docker/standard/upgrade_params.json | 5 +- 23 files changed, 151 insertions(+), 82 deletions(-) delete mode 100644 spec/docker/live_audit.service delete mode 100755 spec/docker/live_audit.sh create mode 100644 spec/docker/pe-old-pub.gpg diff --git a/spec/docker/.dockerignore b/spec/docker/.dockerignore index 335ec957..24e1c028 100644 --- a/spec/docker/.dockerignore +++ b/spec/docker/.dockerignore @@ -1 +1,2 @@ *.tar.gz +puppet-enterprise* diff --git a/spec/docker/Dockerfile b/spec/docker/Dockerfile index 3a2d2705..6072f424 100644 --- a/spec/docker/Dockerfile +++ b/spec/docker/Dockerfile @@ -5,15 +5,15 @@ FROM centos:7 EXPOSE 22 ENV LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8" -RUN echo "LANG=en_US.UTF-8" > /etc/locale.conf +RUN echo "LANG=en_US.UTF-8" > /etc/locale.conf STOPSIGNAL SIGRTMIN+3 -ADD live_audit.sh /usr/bin/live_audit.sh -ADD live_audit.service /etc/systemd/system/live_audit.service -RUN chmod 644 /etc/systemd/system/live_audit.service && chmod 755 /usr/bin/live_audit.sh && \ - echo "root:test" | chpasswd; \ - useradd -m -s /bin/bash centos && echo "centos:test" | chpasswd; -RUN yum -y install epel-release systemd rsync tree vim openssh openssh-server openssh-clients anacron sudo curl openssl -RUN yum -y install inotify-tools && mkdir /root/bolt_scripts && yum clean all +RUN echo "root:test" | chpasswd && \ + useradd -m -s /bin/bash centos && echo "centos:test" | chpasswd; +RUN yum -y install wget systemd rsync tree vim openssh openssh-server openssh-clients anacron sudo curl openssl +RUN yum remove -y puppet-agent # remove any old puppet-agent +COPY pe-old-pub.gpg /tmp +RUN mkdir /root/bolt_scripts && yum clean all && \ + gpg --import /tmp/pe-old-pub.gpg # remove any scripts that don't need to be run RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ rm -f /lib/systemd/system/multi-user.target.wants/*; \ @@ -24,8 +24,7 @@ RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == system rm -f /lib/systemd/system/basic.target.wants/*; \ rm -f /lib/systemd/system/anaconda.target.wants/*; \ rm -rf /var/cache/yum; \ - ln -s '/usr/lib/systemd/system/sshd.service' '/etc/systemd/system/multi-user.target.wants/sshd.service' && \ - ln -s '/etc/systemd/system/live_audit.service' '/etc/systemd/system/multi-user.target.wants/live_audit.service' + ln -s '/usr/lib/systemd/system/sshd.service' '/etc/systemd/system/multi-user.target.wants/sshd.service' ARG HOST="localhost" RUN mkdir -p /root/.puppetlabs/client-tools && \ echo $' \n\ diff --git a/spec/docker/extra-large-ha/docker-compose.yaml b/spec/docker/extra-large-ha/docker-compose.yaml index f0b3792e..1db9e0af 100644 --- a/spec/docker/extra-large-ha/docker-compose.yaml +++ b/spec/docker/extra-large-ha/docker-compose.yaml @@ -66,7 +66,9 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + pe_xl_core_1: restart: always depends_on: @@ -96,7 +98,9 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + pe_xl_core_0: restart: always build: @@ -124,4 +128,6 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + diff --git a/spec/docker/extra-large-ha/params.json b/spec/docker/extra-large-ha/params.json index 23119ddb..6e306b5a 100644 --- a/spec/docker/extra-large-ha/params.json +++ b/spec/docker/extra-large-ha/params.json @@ -3,9 +3,15 @@ "primary_postgresql_host": "pe-xl-db-0.puppet.vm", "replica_postgresql_host": "pe-xl-db-1.puppet.vm", "replica_host": "pe-xl-core-1.puppet.vm", - "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], + "compiler_hosts": [ + "pe-xl-compiler-0.puppet.vm" + ], "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "pe-xl-core-0.puppet.vm", "puppet-xl.vm" ], + "dns_alt_names": [ + "puppet", + "pe-xl-core-0.puppet.vm", + "puppet-xl.vm" + ], "version": "2019.8.5", "compiler_pool_address": "puppet-xl.vm" } diff --git a/spec/docker/extra-large-ha/upgrade_params.json b/spec/docker/extra-large-ha/upgrade_params.json index e3bcb8d7..571f3b3a 100644 --- a/spec/docker/extra-large-ha/upgrade_params.json +++ b/spec/docker/extra-large-ha/upgrade_params.json @@ -1,8 +1,10 @@ { - "primary_host": "pe-xl-core-0.puppet.vm", - "primary_postgresql_host": "pe-xl-db-0.puppet.vm", - "replica_postgresql_host": "pe-xl-db-1.puppet.vm", - "replica_host": "pe-xl-core-1.puppet.vm", - "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], - "version": "2019.8.5" + "primary_host": "pe-xl-core-0.puppet.vm", + "primary_postgresql_host": "pe-xl-db-0.puppet.vm", + "replica_postgresql_host": "pe-xl-db-1.puppet.vm", + "replica_host": "pe-xl-core-1.puppet.vm", + "compiler_hosts": [ + "pe-xl-compiler-0.puppet.vm" + ], + "version": "2019.8.5" } diff --git a/spec/docker/extra-large/docker-compose.yaml b/spec/docker/extra-large/docker-compose.yaml index b6afdb2b..280fa9f1 100644 --- a/spec/docker/extra-large/docker-compose.yaml +++ b/spec/docker/extra-large/docker-compose.yaml @@ -69,4 +69,6 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + diff --git a/spec/docker/extra-large/params.json b/spec/docker/extra-large/params.json index 0be575d5..2278aa6e 100644 --- a/spec/docker/extra-large/params.json +++ b/spec/docker/extra-large/params.json @@ -1,8 +1,13 @@ { "primary_host": "pe-xl-core-0.puppet.vm", "primary_postgresql_host": "pe-xl-db-0.puppet.vm", - "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], + "compiler_hosts": [ + "pe-xl-compiler-0.puppet.vm" + ], "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "pe-xl-core-0.puppet.vm" ], + "dns_alt_names": [ + "puppet", + "pe-xl-core-0.puppet.vm" + ], "version": "2019.8.5" } diff --git a/spec/docker/extra-large/upgrade_params.json b/spec/docker/extra-large/upgrade_params.json index 72204613..e6e23f3a 100644 --- a/spec/docker/extra-large/upgrade_params.json +++ b/spec/docker/extra-large/upgrade_params.json @@ -2,5 +2,5 @@ "primary_host": "pe-xl-core-0.puppet.vm", "primary_postgresql_host": "pe-xl-db-0.puppet.vm", "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], - "version": "2019.8.5" + "version": "2019.8.5" } diff --git a/spec/docker/large-ha/docker-compose.yaml b/spec/docker/large-ha/docker-compose.yaml index 06220f8c..852026d1 100644 --- a/spec/docker/large-ha/docker-compose.yaml +++ b/spec/docker/large-ha/docker-compose.yaml @@ -50,7 +50,8 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + large_aio: build: dockerfile: 'Dockerfile' @@ -78,3 +79,5 @@ services: - /tmp volumes: - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + diff --git a/spec/docker/large-ha/params.json b/spec/docker/large-ha/params.json index 3f26ab19..1645b0b5 100644 --- a/spec/docker/large-ha/params.json +++ b/spec/docker/large-ha/params.json @@ -1,8 +1,13 @@ { "primary_host": "pe-lg.puppet.vm", "replica_host": "pe-lg-replica.puppet.vm", - "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], + "compiler_hosts": [ + "pe-lg-compiler-0.puppet.vm" + ], "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "pe-lg.puppet.vm" ], + "dns_alt_names": [ + "puppet", + "pe-lg.puppet.vm" + ], "version": "2019.8.5" } diff --git a/spec/docker/large-ha/upgrade_params.json b/spec/docker/large-ha/upgrade_params.json index 5a40ff6a..4e6133d7 100644 --- a/spec/docker/large-ha/upgrade_params.json +++ b/spec/docker/large-ha/upgrade_params.json @@ -1,7 +1,8 @@ { - "primary_host": "pe-lg.puppet.vm", - "replica_host": "pe-lg-replica.puppet.vm", - "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], - "version": "2019.8.5" + "primary_host": "pe-lg.puppet.vm", + "replica_host": "pe-lg-replica.puppet.vm", + "compiler_hosts": [ + "pe-lg-compiler-0.puppet.vm" + ], + "version": "2019.8.5" } - \ No newline at end of file diff --git a/spec/docker/large/docker-compose.yaml b/spec/docker/large/docker-compose.yaml index 47594e1a..619d05b9 100644 --- a/spec/docker/large/docker-compose.yaml +++ b/spec/docker/large/docker-compose.yaml @@ -48,4 +48,5 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + diff --git a/spec/docker/large/params.json b/spec/docker/large/params.json index e48365d3..29b5a212 100644 --- a/spec/docker/large/params.json +++ b/spec/docker/large/params.json @@ -1,7 +1,12 @@ { "primary_host": "pe-lg.puppet.vm", - "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], + "compiler_hosts": [ + "pe-lg-compiler-0.puppet.vm" + ], "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "pe-lg.puppet.vm" ], - "version": "2019.8.5" + "dns_alt_names": [ + "puppet", + "pe-lg.puppet.vm" + ], + "version": "2019.8.1" } diff --git a/spec/docker/large/upgrade_params.json b/spec/docker/large/upgrade_params.json index 43cbf7dc..a36c96fe 100644 --- a/spec/docker/large/upgrade_params.json +++ b/spec/docker/large/upgrade_params.json @@ -1,6 +1,7 @@ { - "primary_host": "pe-lg.puppet.vm", - "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], - "version": "2019.8.5" + "primary_host": "pe-lg.puppet.vm", + "compiler_hosts": [ + "pe-lg-compiler-0.puppet.vm" + ], + "version": "2019.8.8" } - \ No newline at end of file diff --git a/spec/docker/live_audit.service b/spec/docker/live_audit.service deleted file mode 100644 index 151df980..00000000 --- a/spec/docker/live_audit.service +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -After=network.service - -[Service] -ExecStart=/bin/bash /usr/bin/live_audit.sh /root/bolt_scripts /tmp/backup - -[Install] -WantedBy=default.target \ No newline at end of file diff --git a/spec/docker/live_audit.sh b/spec/docker/live_audit.sh deleted file mode 100755 index d774023c..00000000 --- a/spec/docker/live_audit.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# must install inotify-tools from epel or self hosted repo -# live_audit.sh /watch_dir /tmp/backup -src=$1 -dst=$2 -logfile=${2}/watcher.log -script_file=${0##*/} -mkdir -p $dst -if [[ ! -d $src || ! -d $dst ]]; then - echo "Source or destanation directory does not exist" - exit 1 -fi -#ps -af -ww |grep [l]ive_backup -while true -do - while inotifywait --outfile=${logfile} -r -e modify,move,close_write,create $src - do - rsync -avz $src/ $dst - done -done \ No newline at end of file diff --git a/spec/docker/pe-old-pub.gpg b/spec/docker/pe-old-pub.gpg new file mode 100644 index 00000000..74024dcf --- /dev/null +++ b/spec/docker/pe-old-pub.gpg @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFe2Iz4BEADqbv/nWmR26bsivTDOLqrfBEvRu9kSfDMzYh9Bmik1A8Z036Eg +h5+TZD8Rrd5TErLQ6eZFmQXk9yKFoa9/C4aBjmsL/u0yeMmVb7/66i+x3eAYGLzV +FyunArjtefZyxq0B2mdRHE8kwl5XGl8015T5RGHCTEhpX14O9yigI7gtliRoZcl3 +hfXtedcvweOf9VrV+t5LF4PrZejom8VcB5CE2pdQ+23KZD48+Cx/sHSLHDtahOTQ +5HgwOLK7rBll8djFgIqP/UvhOqnZGIsg4MzTvWd/vwanocfY8BPwwodpX6rPUrD2 +aXPsaPeM3Q0juDnJT03c4i0jwCoYPg865sqBBrpOQyefxWD6UzGKYkZbaKeobrTB +xUKUlaz5agSK12j4N+cqVuZUBAWcokXLRrcftt55B8jz/Mwhx8kl6Qtrnzco9tBG +T5JN5vXMkETDjN/TqfB0D0OsLTYOp3jj4hpMpG377Q+6D71YuwfAsikfnpUtEBxe +NixXuKAIqrgG8trfODV+yYYWzfdM2vuuYiZW9pGAdm8ao+JalDZss3HL7oVYXSJp +MIjjhi78beuNflkdL76ACy81t2TvpxoPoUIG098kW3xd720oqQkyWJTgM+wV96bD +ycmRgNQpvqHYKWtZIyZCTzKzTTIdqg/sbE/D8cHGmoy0eHUDshcE0EtxsQARAQAB +tEhQdXBwZXQsIEluYy4gUmVsZWFzZSBLZXkgKFB1cHBldCwgSW5jLiBSZWxlYXNl +IEtleSkgPHJlbGVhc2VAcHVwcGV0LmNvbT6JAj4EEwECACgFAle2Iz4CGwMFCQlm +AYAGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH9DgoDvjTSfIN0P/jcCRzK8 +WIdhcNz5dkj7xRZb8Oft2yDfenQmzb1SwGGa96IwJFcjF4Nq7ymcDUqunS2DEDb2 +gCucsqmW1ubkaggsYbc9voz/SQwhsQpBjfWbuyOX9DWmW6av/aB1F85wP79gyfqT +uidTGxQE6EhDbLe7tuvxOHfM1bKsUtI+0n9TALLLHfXUEdtaXCwMlJuO1IIn1PWa +H7HzyEjw6OW/cy73oM9nuErBIio1O60slPLOW2XNhdWZJCRWkcXyuumRjoepz7WN +1JgsLOTcB7rcQaBP3pDN0O/Om5dlDQ6oYitoJs/F0gfEgwK68Uy8k8sUR+FLLJqM +o0CwOg6CeWU4ShAEd1xZxVYW6VOOKlz9x9dvjIVDn2SlTBDmLS99ySlQS57rjGPf +GwlRUnuZP4OeSuoFNNJNb9PO6XFSP66eNHFbEpIoBU7phBzwWpTXNsW+kAcY8Rno +8GzKR/2FRsxe5Nhfh8xy88U7BA0tqxWdqpk/ym+wDcgHBfSRt0dPFnbaHAiMRlgX +J/NPHBQtkoEdQTKA+ICxcNTUMvsPDQgZcU1/ViLMN+6kZaGNDVcPeMgDvqxu0e/T +b3uYiId38HYbHmD6rDrOQL/2VPPXbdGbxDGQUgX1DfdOuFXw1hSTilwI1KdXxUXD +sCsZbchgliqGcI1l2En62+6pI2x5XQqqiJ7+uQINBFe2Iz4BEADzbs8WhdBxBa0t +JBl4Vz0brDgU3YDqNkqnra/T17kVPI7s27VEhoHERmZJ17pKqb2pElpr9mN/FzuN +0N9wvUaumd9gxzsOCam7DPTmuSIvwysk391mjCJkboo01bhuVXe2FBkgOPFzAJEH +YFPxmu7tWOmCxNYiuuYtxLywU7lC/Zp6CZuq57xJqUWK47I5wDK9/iigkwSb3nDs +6A2LpkDmCr+rcOwLh5bxDSei7vYW+3TNOkPlC/h6fO9dPeC9AfyW6qPdVFQq1mpZ +Zcj1ALz7zFiciIB4NrD3PTjDlRnaJCWKPafVSsMbyIWmQaJ01ifuE0Owianrau8c +I264VXmI5pA9C8k9f2aVBuJiLsXaLEb03CzFWz9JpBLttA9ccaam3feU2EmnC3sb +9xD+Ibkxq5mKFN3lEzUAAIqbI1QYGZXPgLxMY7JSvoUxAqeHwpf/dO2LIUqYUpx0 +bF/GWRV9Uql8omNQbhwP0p2X/0Gfxj9Abg2IJM8LeOu3Xk0HACwwyVXgxcgk5FO+ ++KZpTN3iynjmbIzB9qcd9TeSzjVh/RDPSdn5K6Ao5ynubGYmaPwCk+DdVBRDlgWo +7yNIF4N9rFuSMAEJxA1nS5TYFgIN9oDF3/GHngVGfFCv4EG3yS08Hk1tDV0biKdK +ypcx402TAwVRWP5Pzmxc6/ZXU4ZhZQARAQABiQIlBBgBAgAPBQJXtiM+AhsMBQkJ +ZgGAAAoJEH9DgoDvjTSfbWYQALwafIQK9avVNIuhMsyYPa/yHf6rUOLqrYO1GCmj +vyG4cYmryzdxyfcXEmuE5QAIbEKSISrcO6Nvjt9PwLCjR/dUvco0f0YFTPv+kamn ++Bwp2Zt6d3MenXC6mLXPHR4OqFjzCpUT8kFwycvGPsuqZQ/CO0qzLDmAGTY+4ly3 +9aQEsQyFhV3P+6SWnaC2TldWpfG/2pCSaSa8dbYbRe3SUNKXwT8kw3WoQYNofF6n +or8oFVA+UIVlvHc5h7L3tfFylRy5CwtR5rBQtoBicRVxEQc7ARNmB1XWuPntMQl/ +N1Fcfc+KSILFblAR6eVv+6BhMvRqzxqe81AEAP+oKVVwJ7H+wTQun2UKAgZATDWP +/LQsYinmLADpraDPqxT2WJe8kjszMDQZCK+jhsVrhZdkiw9EHAM0z7BKz6JERmLu +TIEcickkTfzbJWXZgv40Bvl99yPMswnR1lQHD7TKxyHYrI7dzJQri4mbORg4lOnZ +3Tyodv21Ocf4as2No1p6esZW+M46zjZeO8zzExmmENI2+P7/VUt+LWyQFiqRM0iW +zGioYMWgVePywFGaTV51/0uF9ymHHC7BDIcLgUWHdg/1B67jR5YQfzPJUqLhnylt +1sjDRQIlf+3U+ddvre2YxX/rYUI2gBT32QzQrv016KsiZO+N+Iya3B4D68s6xxQS +3xJn +=mMjt +-----END PGP PUBLIC KEY BLOCK----- diff --git a/spec/docker/standard-ha/docker-compose.yaml b/spec/docker/standard-ha/docker-compose.yaml index f0663235..600eb8f0 100644 --- a/spec/docker/standard-ha/docker-compose.yaml +++ b/spec/docker/standard-ha/docker-compose.yaml @@ -28,7 +28,8 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + standard_aio: build: dockerfile: 'Dockerfile' @@ -55,4 +56,6 @@ services: - /run - /tmp volumes: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + diff --git a/spec/docker/standard-ha/params.json b/spec/docker/standard-ha/params.json index 09c1911e..ce711c0b 100644 --- a/spec/docker/standard-ha/params.json +++ b/spec/docker/standard-ha/params.json @@ -2,6 +2,9 @@ "primary_host": "pe-std.puppet.vm", "replica_host": "pe-std-replica.puppet.vm", "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "pe-std.puppet.vm" ], + "dns_alt_names": [ + "puppet", + "pe-std.puppet.vm" + ], "version": "2019.8.5" } diff --git a/spec/docker/standard-ha/upgrade_params.json b/spec/docker/standard-ha/upgrade_params.json index bbb1878a..e07b3b2d 100644 --- a/spec/docker/standard-ha/upgrade_params.json +++ b/spec/docker/standard-ha/upgrade_params.json @@ -1,6 +1,5 @@ { - "primary_host": "pe-std.puppet.vm", - "replica_host": "pe-std-replica.puppet.vm", - "version": "2019.8.5" + "primary_host": "pe-std.puppet.vm", + "replica_host": "pe-std-replica.puppet.vm", + "version": "2019.8.5" } - \ No newline at end of file diff --git a/spec/docker/standard/docker-compose.yaml b/spec/docker/standard/docker-compose.yaml index 6892bfe6..913873e6 100644 --- a/spec/docker/standard/docker-compose.yaml +++ b/spec/docker/standard/docker-compose.yaml @@ -31,3 +31,5 @@ services: - /tmp volumes: - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + + diff --git a/spec/docker/standard/params.json b/spec/docker/standard/params.json index ee571338..8663adfd 100644 --- a/spec/docker/standard/params.json +++ b/spec/docker/standard/params.json @@ -1,7 +1,9 @@ { "primary_host": "pe-std.puppet.vm", "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "pe-std.puppet.vm" ], - "version": "2019.8.5", - "r10k_remote": "https://gitlab.com/nwops/control-repo.git" + "dns_alt_names": [ + "puppet", + "pe-std.puppet.vm" + ], + "version": "2019.8.8" } diff --git a/spec/docker/standard/upgrade_params.json b/spec/docker/standard/upgrade_params.json index e5f3eae1..a8556999 100644 --- a/spec/docker/standard/upgrade_params.json +++ b/spec/docker/standard/upgrade_params.json @@ -1,5 +1,4 @@ { - "primary_host": "pe-std.puppet.vm", - "version": "2019.8.5" + "primary_host": "pe-std.puppet.vm", + "version": "2019.8.8" } - \ No newline at end of file From 2313fa72fa58d5cd794d3ce950c19c0c906178c5 Mon Sep 17 00:00:00 2001 From: Corey Osman Date: Thu, 30 Sep 2021 19:06:29 -0700 Subject: [PATCH 2/4] Use the docker host's bolt command and bolt project file * The previously implementation used pdk which ships with an older version of bolt. This allows the user to use the local installation of bolt which is the same procedure as using peadm directly. * This also fixes a few issues around module dependencies and relying on the fixtures path. Users are no longer required to have pdk or run bundle commands. Since this uses pure bolt the entire process mimics that of a real installation and usage. * Updates install and upgrade scripts to use bolt commands --- spec/docker/Puppetfile | 13 +++++++++++++ spec/docker/bolt-project.yaml | 12 ++++++++++++ spec/docker/install.sh | 19 +++++++------------ spec/docker/upgrade.sh | 16 +++++----------- 4 files changed, 37 insertions(+), 23 deletions(-) create mode 100644 spec/docker/Puppetfile create mode 100644 spec/docker/bolt-project.yaml diff --git a/spec/docker/Puppetfile b/spec/docker/Puppetfile new file mode 100644 index 00000000..99608929 --- /dev/null +++ b/spec/docker/Puppetfile @@ -0,0 +1,13 @@ +# This Puppetfile is managed by Bolt. Do not edit. +# For more information, see https://pup.pt/bolt-modules + +# The following directive installs modules to the managed moduledir. +moduledir '.modules' + +mod 'puppet/format', '1.0.0' +mod 'puppetlabs/bolt_shim', '0.3.2' +mod 'puppetlabs/apply_helpers', '0.3.0' +mod 'WhatsARanjit/node_manager', '0.7.5' +mod 'puppetlabs/ruby_task_helper', '0.5.1' +mod 'puppetlabs/stdlib', '7.1.0' +mod 'nwops/container_inventory', '0.1.1' diff --git a/spec/docker/bolt-project.yaml b/spec/docker/bolt-project.yaml new file mode 100644 index 00000000..01f787a2 --- /dev/null +++ b/spec/docker/bolt-project.yaml @@ -0,0 +1,12 @@ +--- +name: peadm_docker_examples +modules: + - name: nwops/container_inventory + version_requirement: ">= 0.1.1" + - name: puppetlabs/stdlib + version_requirement: ">= 6.5.0 < 8.0.0" + - puppetlabs/ruby_task_helper + - WhatsARanjit/node_manager + - puppetlabs/apply_helpers + - puppetlabs/bolt_shim + - puppet/format diff --git a/spec/docker/install.sh b/spec/docker/install.sh index b494cca0..d5ea98c5 100755 --- a/spec/docker/install.sh +++ b/spec/docker/install.sh @@ -1,31 +1,26 @@ #!/usr/bin/env bash -# bundle install or pdk bundle install -# bundle exec rake spec_prep or pdk bundle exec rake spec_prep # must be in the spec/docker directory -echo 'Please choose a PE architecture to provision: ' +# must have bolt 3.18+ installed downloads=$(realpath ./) inventory_dir=$(realpath ./) inventory_path=${inventory_dir}/inventory.yaml base_repo=$(realpath ../../) spec_path=$(realpath ../) -fixtures_path=$spec_path/fixtures/modules -num=$(ls ${fixtures_path} | wc -l) -if [[ ! "$num" -gt "8" ]]; then - echo "No fixtures, please run bundle exec rake spec_prep or pdk bundle exec rake spec_prep" - exit 1 -fi +bolt module install +ln -nfs ../../../ ./.modules/peadm # The concurrency is set to 2 to keep CPU usage from skyrocketing during Large and XL deployments +echo 'Please choose a PE architecture to provision: ' + select opt in */ do dir=$(realpath ${opt}) name=$(basename $opt) cd $dir docker-compose up -d --build - # nohup /usr/bin/live_audit.sh /root/bolt_scripts /tmp/backup & - pdk bundle exec bolt plan run peadm::install --concurrency 2 \ + bolt plan run peadm::install --concurrency 2 \ --inventory $inventory_path \ - --modulepath=$fixtures_path \ --params @${dir}/params.json \ --targets=$name break; done +# --modulepath=./modules \ diff --git a/spec/docker/upgrade.sh b/spec/docker/upgrade.sh index ab621b18..a6b17a19 100755 --- a/spec/docker/upgrade.sh +++ b/spec/docker/upgrade.sh @@ -1,19 +1,14 @@ #!/usr/bin/env bash -# bundle install or pdk bundle install -# bundle exec rake spec_prep or pdk bundle exec rake spec_prep # must be in the spec/docker directory +# must have bolt 3.18+ installed echo 'Please choose a PE architecture to provision: ' downloads=$(realpath ./) inventory_dir=$(realpath ./) inventory_path=${inventory_dir}/inventory.yaml base_repo=$(realpath ../../) -spec_path=$(realpath ../) -fixtures_path=$spec_path/fixtures/modules -num=$(ls ${fixtures_path} | wc -l) -if [[ ! "$num" -gt "8" ]]; then - echo "No fixtures, please run bundle exec rake spec_prep or pdk bundle exec rake spec_prep" - exit 1 -fi +bolt module install +# bolt will clobber the .modules directory so a new link is required +ln -nfs ../../../ ./.modules/peadm # The concurrency is set to 2 to keep CPU usage from skyrocketing during Large and XL deployments select opt in */ do @@ -22,9 +17,8 @@ do cd $dir docker-compose up -d --build # nohup /usr/bin/live_audit.sh /root/bolt_scripts /tmp/backup & - pdk bundle exec bolt plan run peadm::upgrade --concurrency 2 \ + bolt plan run peadm::upgrade --concurrency 2 \ --inventory $inventory_path \ - --modulepath=$fixtures_path \ --params @${dir}/upgrade_params.json \ --targets=$name break; From f81a6359e588b61de9ee25b1ce5cbaf69944c137 Mon Sep 17 00:00:00 2001 From: Corey Osman Date: Wed, 6 Oct 2021 13:08:31 -0700 Subject: [PATCH 3/4] Ensure we don't include generated files into git --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 3aabfaed..aa93de59 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ .plan_cache.json .resource_types/ bolt-debug.log +spec/docker/**/*.tar.gz +spec/docker/**/*.asc +spec/docker/**/files/puppet-enterprise* +spec/docker/.task_cache.json From abfd2b88777b0801b79fcf22e93b815161632695 Mon Sep 17 00:00:00 2001 From: Corey Osman Date: Wed, 6 Oct 2021 13:09:37 -0700 Subject: [PATCH 4/4] Adds ability to optionally add agents for docker examples * Previously the documentation detailed instructions for how to add agents to a docker example cluster. This adds a script to do the dirty work. --- documentation/docker_examples.md | 45 +++++++++++++++++++++++++++----- spec/docker/setup-agents.sh | 39 +++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 spec/docker/setup-agents.sh diff --git a/documentation/docker_examples.md b/documentation/docker_examples.md index c61aefc4..7efc1a7e 100644 --- a/documentation/docker_examples.md +++ b/documentation/docker_examples.md @@ -1,17 +1,12 @@ ## Docker Based Examples This module provides docker compose files for the various architectures for experimentation purposes. This gives you the ability to stand up an entire PE stack in order to learn how this module and DR works. If you have docker and docker-compose you can start up a full Puppet architecture with a single command. Please note that Puppet does not support PE on containers in production. -In order to decouple Bolt from a dev system, a special bolt container is created that will run all the bolt commands. This is -required to achieve maximum portability. Should you want to run bolt commands against the PE stack you must -first login to this bolt container via ssh, docker or docker-compose. - -Example: `docker-compose run --entrypoint=/bin/bash bolt` - ### Requirements To run the container based examples you will need the following requirements: 1. Docker 2. Docker compose +3. Bolt 3.18+ 3. realpath (brew install coreutils on mac) 4. 16GB memory, 24GB+ for XL and XL-DR architectures 5. CPU with many cores (Tested with Core i7 6700) @@ -46,6 +41,8 @@ In order to stop and remove the containers you will need to perform the followin 2. `cd ` 3. docker-compose down +Alternative you can run something similar like: `docker-compose --project-directory=large down` + ### Logging into the console You can login to the PE Console after successful install. However, first you will need to grab the mapped port number of the PE console. The port numbers are mapped dynamically as to not @@ -108,6 +105,8 @@ docker inspect pe-xl-core-0.puppet.vm -f "{{json .NetworkSettings.Networks }}" | } ``` +You can also run `docker inspect pe-xl-core-0.puppet.vm -f "{{json .HostConfig.NetworkMode }}" | sed -e 's/^"//' -e 's/"$//'` + **NOTE** In these example you may see the use of `jq`. This is a [cli utility for parsing JSON](https://stedolan.github.io/jq/). I recommend installing it. As a alternative you can pipe output to `python -m json.tool`. ### Starting agent containers @@ -122,8 +121,38 @@ Example: For most tasks these images are great. However, if you wish to use puppet orchestrator with the pcp transport. The one requirement is that all images used must be systemd aware, otherwise pxp will not start. If you do not plan on using pcp there is no need for containers with systemd. -At this time we have not added documention for starting a container with systemd. Instructions coming soon. +You can use the the custom image `pe-base` built with the docker cluster named pe-base. This is a centos:7 image that you can use to generate tens or hundreds of agents. (Resources permitting). You will also want to run the docker run command with additonal flags. + +`docker run -d -t --hostname=pe-std-agent1.puppet.vm --network=pe-std_default --privileged --label=pe-std-agent,docker-example-agent" -v /sys/fs/cgroup:/sys/fs/cgroup:ro pe-base"` + +Once you have created the container you will obviously want to install the puppet agent + +`docker exec -ti $CONTAINER_ID sh -c "curl -k https://${PE_SERVER}:8140/packages/current/install.bash | bash && puppet agent -t"` + +Accept the cert in the console and run the puppet agent again on the agent container. +Login interactively if you wish: `docker exec -ti $CONTAINER_ID /bin/bash` + +Take a look at the spec/docker/Dockerfile for examples of how to setup systemd in a container. + + +### Cleaning up +Before you run docker-compose down inside the cluster type directory you will need to stop and remove +all the agent containers if created. + +This can be done like so: + +```bash +# base name is the name of the primary hostname without domain +PE_CLUSTER_TYPE=standard +BASE_NAME=pe-std +docker stop $(docker ps -q -f label="${BASE_NAME}-agent") +docker rm $(docker ps -a -q -f label="${BASE_NAME}-agent") +# The docker-compose down command cannot be run until the agents have been removed +cd spec/docker/${PE_CLUSTER_TYPE} +docker-compose down + +``` ### Other notes 1. The install plan is not fully idempotent. @@ -134,3 +163,5 @@ At this time we have not added documention for starting a container with systemd 6. You can use top to view all the processes being run in the containers. 7. Docker will use the privilege mode option when running these examples (systemd support) 8. Systemd is running inside these containers! The real systemd, not the fake one. + +If you see errors regarding peadmin::puppet_runonce, just run the install or upgrade script again. Might have to perform multiple times for resource constrained docker hosts. diff --git a/spec/docker/setup-agents.sh b/spec/docker/setup-agents.sh new file mode 100644 index 00000000..dadaf03c --- /dev/null +++ b/spec/docker/setup-agents.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Purpose: Create container agents for docker examples +PE_SERVER=$1 +MAX_AGENTS="${2:-5}" +if [[ -z $PE_SERVER ]]; then + echo "No pe server provided, please provide the fqdn of your primary servername" + echo "Example usage: $0 pe-std.puppet.vm [num agent containers]" + echo "The default number of agent containers is 5" + exit 1 +fi +SRV_CMD="echo ${PE_SERVER} | cut -d. -f1" +BASE_NAME=$(eval $SRV_CMD) +DOCKER_NETWORK=$(docker inspect ${PE_SERVER} -f "{{json .HostConfig.NetworkMode }}" | sed -e 's/^"//' -e 's/"$//') +if [[ -z $DOCKER_NETWORK ]]; then + echo "docker network not found for ${PE_SERVER}, exiting" + exit 1 +fi +# start loop here +for (( i=1; i<=$MAX_AGENTS; i++ )) +do + # need a way better way come up with a unique hostname + AGENT_HOSTNAME="${BASE_NAME}-agent-${i}.puppet.vm" + #--name $AGENT_HOSTNAME --hostname=$AGENT_HOSTNAME could be usedbut we will get duplicate certs without cleaning on ca + INSTALL_CMD="curl -k https://${PE_SERVER}:8140/packages/current/install.bash | bash" + RUN_CMD="docker run -d -t --network=${DOCKER_NETWORK} --privileged --label=\"${BASE_NAME}-agent\" --label=\"docker-example-agent\" -v /sys/fs/cgroup:/sys/fs/cgroup:ro pe-base" + echo RUN_CMD + CONTAINER=$(eval $RUN_CMD) + CONTAINER=${CONTAINER:0:12} + if [[ -z $CONTAINER ]]; then + echo "Container was not started for some reason" + exit 1 + fi + SETUP="docker exec -ti $CONTAINER sh -c \"${INSTALL_CMD} && puppet agent -t\"" + eval $SETUP + CHOST=$(docker exec $CONTAINER /opt/puppetlabs/bin/puppet config print certname) + # if user manually signs certs, we need to fail gracefully + docker exec -ti $PE_SERVER sh -c "/opt/puppetlabs/bin/puppetserver ca sign --certname ${CHOST}" + docker exec -ti $CONTAINER sh -c "puppet agent -t" +done