From 9f24023445f8b056c4cb1483256ffd6090e32dbd Mon Sep 17 00:00:00 2001 From: Reid Vandewiele Date: Tue, 31 Mar 2020 11:20:13 -0700 Subject: [PATCH] Reduce several check tasks down to one This commit consolidates two tasks which check for a service to be available into one, and provides a convenience function wrapper for using that task so that the high-level plans are easier to read. Right now every use of the check just waits until the service is running, so the implementation puts the wait logic in the task itself. In the event a future need to actually return true/false if the service is running or not, it will be easy to refactor into a single "service_status" task instead, and put the wait logic into the convenience function wrapper for today's "wait until" uses. --- functions/wait_until_service_ready.pp | 16 ++++++++++++++++ plans/action/install.pp | 16 ++++------------ plans/upgrade.pp | 4 ++-- tasks/check_status.json | 13 ------------- tasks/check_status.sh | 18 ------------------ tasks/orchestrator_healthcheck.json | 8 -------- tasks/orchestrator_healthcheck.sh | 22 ---------------------- tasks/wait_until_service_ready.json | 16 ++++++++++++++++ tasks/wait_until_service_ready.sh | 17 +++++++++++++++++ 9 files changed, 55 insertions(+), 75 deletions(-) create mode 100644 functions/wait_until_service_ready.pp delete mode 100644 tasks/check_status.json delete mode 100644 tasks/check_status.sh delete mode 100644 tasks/orchestrator_healthcheck.json delete mode 100644 tasks/orchestrator_healthcheck.sh create mode 100644 tasks/wait_until_service_ready.json create mode 100644 tasks/wait_until_service_ready.sh diff --git a/functions/wait_until_service_ready.pp b/functions/wait_until_service_ready.pp new file mode 100644 index 00000000..f100afe0 --- /dev/null +++ b/functions/wait_until_service_ready.pp @@ -0,0 +1,16 @@ +# A convenience function to help remember port numbers for services and handle +# running the wait_until_service_ready task +function peadm::wait_until_service_ready( + String $service, + TargetSpec $target, +) { + $port = case $service { + 'orchestrator-service': { '8143' } + default: { '8140' } + } + + run_task('peadm::wait_until_service_ready', $target, + service => $service, + port => $port, + ) +} diff --git a/plans/action/install.pp b/plans/action/install.pp index 6f84a1c2..8bb6136d 100644 --- a/plans/action/install.pp +++ b/plans/action/install.pp @@ -330,19 +330,11 @@ ) } + # The puppetserver might be in the middle of a restart after the Puppet run, + # so we check the status by calling the api and ensuring the puppetserver is + # taking requests before proceeding. run_task('peadm::puppet_runonce', $master_target) - - # The puppetserver might be in the middle of a restart so we check the status by calling - # the api and ensuring the puppetserver is taking requests. - ctrl::do_until('limit' => 10) || { - $pe_status = run_task('peadm::check_status', $master_target, service => 'pe-master') - if ($pe_status.first['state'] != 'running') { - ctrl::sleep(5) - false - } else { - true - } - } + peadm::wait_until_service_ready('pe-master', $master_target) run_task('peadm::puppet_runonce', $all_targets - $master_target) diff --git a/plans/upgrade.pp b/plans/upgrade.pp index 4ee01738..4fd59a7a 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -121,7 +121,7 @@ # If in use, wait until orchestrator service is healthy to proceed if $all_targets.any |$target| { $target.protocol == 'pcp' } { - run_task('peadm::orchestrator_healthcheck', $master_target) + peadm::wait_until_service_ready('orchestrator-service', $master_target) wait_until_available($all_targets, wait_time => 120) } @@ -135,7 +135,7 @@ # The master could restart orchestration services again, in which case we # would have to wait for nodes to reconnect if $all_targets.any |$target| { $target.protocol == 'pcp' } { - run_task('peadm::orchestrator_healthcheck', $master_target) + peadm::wait_until_service_ready('orchestrator-service', $master_target) wait_until_available($all_targets, wait_time => 120) } diff --git a/tasks/check_status.json b/tasks/check_status.json deleted file mode 100644 index 93c7208f..00000000 --- a/tasks/check_status.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "description": "Get the status of a Puppet Service in JSON format", - "parameters": { - "service": { - "type": "Enum[all, ca, code-manager-service, file-sync-client-service,file-sync-storage-service, jruby-metrics, master,pe-jruby-metrics, pe-master, pe-puppet-profiler,puppet-profiler, status-service]", - "description": "What service to check. For example: 'all, status-service, pe-master, master, pe-jruby-metrics code-manager-service'" - } - }, - "input_method": "environment", - "implementations": [ - {"name": "check_status.sh"} - ] -} diff --git a/tasks/check_status.sh b/tasks/check_status.sh deleted file mode 100644 index 25bd543d..00000000 --- a/tasks/check_status.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# For more information about what this task can return see the API -# https://puppet.com/docs/puppetserver/latest/status-api/v1/services.html -# Note: if you pass in a service name you will get only the status of that -# service. -# This task is meant to be run on the puppetserver which is why localhost -# is used as the hostname. -service=$PT_service -if [[ $service == 'all' ]]; then - out=$(curl https://localhost:8140/status/v1/services -k -s) -else - out=$(curl -k -s https://localhost:8140/status/v1/services/${service}) -fi - -code=$? -echo $out -exit $code diff --git a/tasks/orchestrator_healthcheck.json b/tasks/orchestrator_healthcheck.json deleted file mode 100644 index 020fb2f3..00000000 --- a/tasks/orchestrator_healthcheck.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Return when the orchestrator service is healthy, or timeout after 15 seconds", - "parameters": { }, - "input_method": "environment", - "implementations": [ - {"name": "orchestrator_healthcheck.sh"} - ] -} diff --git a/tasks/orchestrator_healthcheck.sh b/tasks/orchestrator_healthcheck.sh deleted file mode 100644 index f7cbcaf5..00000000 --- a/tasks/orchestrator_healthcheck.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -g_certname=$(/opt/puppetlabs/bin/puppet config print certname) - -check() -{ - curl "https://${g_certname}:8143/status/v1/simple" \ - --cert "/etc/puppetlabs/puppet/ssl/certs/${g_certname}.pem" \ - --key "/etc/puppetlabs/puppet/ssl/private_keys/${g_certname}.pem" \ - --cacert "/etc/puppetlabs/puppet/ssl/certs/ca.pem" \ - --silent \ - --show-error \ - --fail -} - -n=0 -until [ $n -ge 10 ] -do - check && break - n=$[$n+1] - sleep 3 -done diff --git a/tasks/wait_until_service_ready.json b/tasks/wait_until_service_ready.json new file mode 100644 index 00000000..3654a984 --- /dev/null +++ b/tasks/wait_until_service_ready.json @@ -0,0 +1,16 @@ +{ + "description": "Return when the orchestrator service is healthy, or timeout after 15 seconds", + "parameters": { + "service": { + "type": "Enum[all, ca, pe-master, orchestrator-service]", + "description": "What service to check. For example: all, pe-master, orchestrator-service" + }, + "port": { + "type": "Enum['8140', '8143']", + "description": "Which port to query the status API on" + } + }, + "implementations": [ + {"name": "wait_until_service_ready.sh", "input_method": "environment"} + ] +} diff --git a/tasks/wait_until_service_ready.sh b/tasks/wait_until_service_ready.sh new file mode 100644 index 00000000..95aceab4 --- /dev/null +++ b/tasks/wait_until_service_ready.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +check() { + if [[ "$PT_service" == 'all' ]]; then + out=$(curl -fks https://localhost:${PT_port}/status/v1/simple) + else + out=$(curl -fks https://localhost:${PT_port}/status/v1/simple/${PT_service}) + fi +} + +n=0 +until [ $n -ge 20 ] +do + check && break + n=$[$n+1] + sleep 3 +done