Merge pull request #81 from appuio/feat/silence-handle-delayed-worker-pools

Add support for creating a separate silence for delayed machineconfigpool maintenance

Author: simu

.cruft.json

@@ -7,7 +7,7 @@
       "name": "openshift-upgrade-controller",
       "slug": "openshift-upgrade-controller",
       "parameter_key": "openshift_upgrade_controller",
-      "test_cases": "defaults",
+      "test_cases": "defaults delayed-pool-silence",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",
@@ -24,7 +24,8 @@
       "github_owner": "appuio",
       "github_name": "component-openshift-upgrade-controller",
       "github_url": "https://github.com/appuio/component-openshift-upgrade-controller",
-      "_template": "https://github.com/projectsyn/commodore-component-template.git"
+      "_template": "https://github.com/projectsyn/commodore-component-template.git",
+      "_commit": "98d16f99766e6c6d97322dbe42e058f0e2bf73d0"
     }
   },
   "directory": null

.github/workflows/test.yaml

@@ -33,6 +33,7 @@ jobs:
       matrix:
         instance:
           - defaults
+          - delayed-pool-silence
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -48,6 +49,7 @@ jobs:
       matrix:
         instance:
           - defaults
+          - delayed-pool-silence
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -50,4 +50,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml
+test_instances = tests/defaults.yml tests/delayed-pool-silence.yml

class/defaults.yml

@@ -44,6 +44,7 @@ parameters:
       alert_matchers: {}
       silence_timeout_hours: 12
       silence_after_finish_minutes: 30
+      handle_delayed_worker_pools: false
       additional_job_configuration:
         metadata: {}
         spec:

component/scripts/silence.sh

@@ -3,12 +3,23 @@ set -xeuo pipefail
 
 job_name="$JOB_metadata_name"
 
+if [ "${EVENT_name}" = "\"MachineConfigPoolUnpause\"" ] && [ "${EVENT_reason}" = "\"Completed\"" ]; then
+  echo "Upgrade completed without MCP upgrade. Not creating a new silence when unpausing MCPs."
+  exit 0
+fi
+
 curl_opts=( "https://${ALERTMANAGER_HOST}.${ALERTMANAGER_NAMESPACE}.svc.cluster.local:9095/api/v2/silences" --cacert /etc/ssl/certs/serving-certs/service-ca.crt --header 'Content-Type: application/json' --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" --resolve "${ALERTMANAGER_HOST}.${ALERTMANAGER_NAMESPACE}.svc.cluster.local:9095:$(getent hosts "${ALERTMANAGER_OPERATED_SERVICE}.${ALERTMANAGER_NAMESPACE}.svc.cluster.local" | awk '{print $1}' | head -n 1)" --silent )
 
 startsAt="$(date -u +'%Y-%m-%dT%H:%M:%S' --date '-5 minutes')"
+# We use SILENCE_TIMEOUT_HOURS regardless of whether we create a silence for
+# the initial maintenance or for a delayed worker pool maintenance, when we
+# trigger on `UpgradeComplete` and `MachineConfigPoolUnpause`.
 endsAt="$(date -u +'%Y-%m-%dT%H:%M:%S' --date "+${SILENCE_TIMEOUT_HOURS} hours")"
 
-if [ "${EVENT_name}" = "\"Finish\"" ]; then
+# Expire silence on Finish or UpgradeComplete events. Also use
+# SILENCE_AFTER_FINISH_MINUTES when expiring silence before paused pools are
+# updated.
+if [ "${EVENT_name}" = "\"Finish\"" ] || [ "${EVENT_name}" = "\"UpgradeComplete\"" ]; then
   endsAt="$(date -u +'%Y-%m-%dT%H:%M:%S' --date "+${SILENCE_AFTER_FINISH_MINUTES} minutes")"
 fi
 

component/silence.libsonnet

@@ -81,13 +81,20 @@ local certcm = kube.ConfigMap('maintenance-silence-certs') + namespace {
   data:: {},
 };
 
+local events = if params.upgrade_silence.handle_delayed_worker_pools then [
+  'Start',
+  'UpgradeComplete',
+  'MachineConfigPoolUnpause',
+  'Finish',
+] else [
+  'Start',
+  'Finish',
+];
+
 local ujh = kube._Object('managedupgrade.appuio.io/v1beta1', 'UpgradeJobHook', 'maintenance-silence') + namespace {
   spec+: {
     selector: params.upgrade_silence.upgrade_job_selector,
-    events: [
-      'Start',
-      'Finish',
-    ],
+    events: events,
     template+: {
       spec+: {
         template+: {

docs/modules/ROOT/pages/references/parameters.adoc

@@ -276,6 +276,26 @@ default:: `30`
 
 The duration to wait after the upgrade job has finished before expiring the silence in minutes.
 
+=== `upgrade_silence.handle_delayed_worker_pools`
+
+[horizontal]
+type:: bool
+default:: `false`
+
+Whether to create separate silences for the initial maintenance and the delayed maintenance of one or more MachineConfigPools.
+
+If set to true, the upgrade silence `UpgradeJobHook` is executed for events `UpgradeComplete` (when the upgrade is complete except for delayed MachineConfigPools) and `MachineConfigPoolUnpause` (when the delayed MachineConfigPool maintenance starts) in addition to the `Start` and `Finish` events.
+
+When the hook script runs for the `UpgradeComplete` event it expires the silence after `upgrade_silence.silence_after_finish_minutes`.
+When the hook script runs for the `MachineConfigPoolUnpause` event it creates a silence which ends after `upgrade_silence.silence_timeout_hours`.
+
+The `Start` and `Finish` logic remains unchanged, the `Finish` run will expire the silence created by the run for `MachineConfigPoolUnpause` when the `handle_delayed_worker_pools` parameter is `true`.
+
+[NOTE]
+====
+Currently, the upgrade-controller doesn't have an event for `MachineConfigPoolComplete`.
+Enabling this parameter may produce undesired silences for configurations which have different delays for multiple delayed MachineConfigPools.
+====
 
 === `upgrade_silence.additional_job_configuration`
 

tests/delayed-pool-silence.yml

@@ -0,0 +1,39 @@
+parameters:
+  openshift_upgrade_controller:
+    upgrade_configs:
+      appuio-monday-afternoon:
+        spec:
+          maxSchedulingDelay: 1h
+          maxUpgradeStartDelay: 1h
+          schedule:
+            cron: "0 10 * * 2"
+            location: Europe/Zurich
+          jobTemplate:
+            metadata:
+              labels:
+                upgradeconfig/name: appuio-monday-afternoon
+            spec:
+              config:
+                upgradeTimeout: 12h
+                preUpgradeHealthChecks:
+                  timeout: 1h
+                postUpgradeHealthChecks:
+                  timeout: 1h
+    upgrade_silence:
+      upgrade_job_selector:
+        matchLabels: ${openshift_upgrade_controller:upgrade_configs:appuio-monday-afternoon:spec:jobTemplate:metadata:labels}
+      alert_matchers:
+        "only maintenance without SLOs":
+          matchers:
+            - name: alertname
+              value: "Watchdog"
+              isRegex: false
+              isEqual: false
+            - name: Maintenance
+              value: "true"
+              isRegex: false
+              isEqual: false
+      additional_job_configuration:
+        metadata: {}
+        spec: {}
+      handle_delayed_worker_pools: true

tests/golden/defaults/openshift-upgrade-controller/openshift-upgrade-controller/90_upgrade_silence.yaml

@@ -22,6 +22,7 @@ rules:
     resources:
       - alertmanagers/api
     verbs:
+      - create
       - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -66,12 +67,23 @@ data:
 
     job_name="$JOB_metadata_name"
 
+    if [ "${EVENT_name}" = "\"MachineConfigPoolUnpause\"" ] && [ "${EVENT_reason}" = "\"Completed\"" ]; then
+      echo "Upgrade completed without MCP upgrade. Not creating a new silence when unpausing MCPs."
+      exit 0
+    fi
+
     curl_opts=( "https://${ALERTMANAGER_HOST}.${ALERTMANAGER_NAMESPACE}.svc.cluster.local:9095/api/v2/silences" --cacert /etc/ssl/certs/serving-certs/service-ca.crt --header 'Content-Type: application/json' --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" --resolve "${ALERTMANAGER_HOST}.${ALERTMANAGER_NAMESPACE}.svc.cluster.local:9095:$(getent hosts "${ALERTMANAGER_OPERATED_SERVICE}.${ALERTMANAGER_NAMESPACE}.svc.cluster.local" | awk '{print $1}' | head -n 1)" --silent )
 
     startsAt="$(date -u +'%Y-%m-%dT%H:%M:%S' --date '-5 minutes')"
+    # We use SILENCE_TIMEOUT_HOURS regardless of whether we create a silence for
+    # the initial maintenance or for a delayed worker pool maintenance, when we
+    # trigger on `UpgradeComplete` and `MachineConfigPoolUnpause`.
     endsAt="$(date -u +'%Y-%m-%dT%H:%M:%S' --date "+${SILENCE_TIMEOUT_HOURS} hours")"
 
-    if [ "${EVENT_name}" = "\"Finish\"" ]; then
+    # Expire silence on Finish or UpgradeComplete events. Also use
+    # SILENCE_AFTER_FINISH_MINUTES when expiring silence before paused pools are
+    # updated.
+    if [ "${EVENT_name}" = "\"Finish\"" ] || [ "${EVENT_name}" = "\"UpgradeComplete\"" ]; then
       endsAt="$(date -u +'%Y-%m-%dT%H:%M:%S' --date "+${SILENCE_AFTER_FINISH_MINUTES} minutes")"
     fi
 

tests/golden/delayed-pool-silence/openshift-upgrade-controller/apps/openshift-upgrade-controller.yaml

@@ -0,0 +1,53 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: openshift-upgrade-controller
+  name: openshift-upgrade-controller
+  namespace: appuio-openshift-upgrade-controller
+spec:
+  groups:
+    - name: drain.alerts
+      rules:
+        - alert: MaintenanceInProgress
+          annotations:
+            description: Cluster is currently upgrading
+            message: An OpenShift upgrade is in progress on this cluster
+            summary: Cluster is currently upgrading
+          expr: |
+            max(max_over_time(openshift_upgrade_controller_upgradejob_state{state="active"} [10m])) > 0
+          for: 0m
+          labels:
+            Maintenance: 'true'
+            severity: info
+            syn: 'true'
+            syn_component: openshift-upgrade-controller
+        - alert: NodeDrainStuck
+          annotations:
+            description: Node {{$labels.node}} is draining for more than 10 minutes.
+            message: Node {{$labels.node}} is draining for more than 10 minutes.
+            runbook_url: https://hub.syn.tools/openshift-upgrade-controller/runbooks/NodeDrainStuck.html
+            summary: Node is draining for more than 10 minutes.
+          expr: |
+            openshift_upgrade_controller_node_draining == 1
+          for: 15m
+          labels:
+            Maintenance: 'true'
+            severity: warning
+            syn: 'true'
+            syn_component: openshift-upgrade-controller
+        - alert: PausedMachineConfigPool
+          annotations:
+            description: |
+              MachineConfigPool {{$labels.pool}} is paused. A paused MachineConfigPool will likely block the next maintenance.
+            message: MachineConfigPool {{$labels.pool}} is paused.
+            runbook_url: https://hub.syn.tools/openshift-upgrade-controller/runbooks/PausedMachineConfigPool.html
+            summary: Paused MachineConfigPool
+          expr: |
+            group(openshift_upgrade_controller_machine_config_pools_paused > 0) by (pool) unless on() group(openshift_upgrade_controller_upgradejob_state{state=~"active|paused"})
+          for: 2h
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift-upgrade-controller

tests/golden/delayed-pool-silence/openshift-upgrade-controller/openshift-upgrade-controller/10_prometheusrule.yaml

@@ -0,0 +1,25 @@
+apiVersion: managedupgrade.appuio.io/v1beta1
+kind: UpgradeConfig
+metadata:
+  annotations: {}
+  labels:
+    name: appuio-monday-afternoon
+  name: appuio-monday-afternoon
+  namespace: appuio-openshift-upgrade-controller
+spec:
+  jobTemplate:
+    metadata:
+      labels:
+        upgradeconfig/name: appuio-monday-afternoon
+    spec:
+      config:
+        postUpgradeHealthChecks:
+          timeout: 1h
+        preUpgradeHealthChecks:
+          timeout: 1h
+        upgradeTimeout: 12h
+  maxSchedulingDelay: 1h
+  maxUpgradeStartDelay: 1h
+  schedule:
+    cron: 0 10 * * 2
+    location: Europe/Zurich

tests/golden/delayed-pool-silence/openshift-upgrade-controller/openshift-upgrade-controller/20_upgradeconfigs.yaml

@@ -0,0 +1,46 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations: {}
+  labels:
+    name: syn-openshift-upgrade-controller-view
+    rbac.authorization.k8s.io/aggregate-to-admin: 'true'
+    rbac.authorization.k8s.io/aggregate-to-edit: 'true'
+    rbac.authorization.k8s.io/aggregate-to-view: 'true'
+  name: syn:openshift-upgrade-controller:view
+rules:
+  - apiGroups:
+      - managedupgrade.appuio.io
+    resources:
+      - clusterversions
+      - upgradeconfigs
+      - upgradejobs
+      - upgradejobhooks
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations: {}
+  labels:
+    name: syn-openshift-upgrade-controller-edit
+    rbac.authorization.k8s.io/aggregate-to-admin: 'true'
+    rbac.authorization.k8s.io/aggregate-to-edit: 'true'
+  name: syn:openshift-upgrade-controller:edit
+rules:
+  - apiGroups:
+      - managedupgrade.appuio.io
+    resources:
+      - clusterversions
+      - upgradeconfigs
+      - upgradejobs
+      - upgradejobhooks
+    verbs:
+      - create
+      - delete
+      - deletecollection
+      - patch
+      - update