Merge pull request #1090 from dgrisonnet/bz-1921335

Bug 1921335: Fix and adjust ThanosSidecarUnhealthy alert

Author: openshift-merge-robot

Diff for: assets/prometheus-k8s/prometheus-rule.yaml

@@ -239,24 +239,35 @@ spec:
       for: 15m
       labels:
         severity: critical
-  - name: thanos-sidecar.rules
+  - name: thanos-sidecar
     rules:
     - alert: ThanosSidecarPrometheusDown
       annotations:
-        description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} cannot connect
+        description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} cannot connect
           to Prometheus.
         summary: Thanos Sidecar cannot connect to Prometheus
       expr: |
-        sum by (job, pod) (thanos_sidecar_prometheus_up{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"} == 0)
-      for: 5m
+        sum by (job, instance) (thanos_sidecar_prometheus_up{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"} == 0)
+      for: 1h
       labels:
-        severity: critical
+        severity: warning
+    - alert: ThanosSidecarBucketOperationsFailed
+      annotations:
+        description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} bucket operations
+          are failing
+        summary: Thanos Sidecar bucket operations are failing
+      expr: |
+        rate(thanos_objstore_bucket_operation_failures_total{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"}[5m]) > 0
+      for: 1h
+      labels:
+        severity: warning
     - alert: ThanosSidecarUnhealthy
       annotations:
         description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for
-          {{ $value }} seconds.
+          more than {{ $value }} seconds.
         summary: Thanos Sidecar is unhealthy.
       expr: |
-        time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"}) by (job, pod) >= 600
+        time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"})) by (job,pod) >= 240
+      for: 1h
       labels:
-        severity: critical
+        severity: warning

Diff for: assets/thanos-querier/prometheus-rule.yaml

@@ -10,7 +10,7 @@ metadata:
   namespace: openshift-monitoring
 spec:
   groups:
-  - name: thanos-query.rules
+  - name: thanos-query
     rules:
     - alert: ThanosQueryHttpRequestQueryErrorRateHigh
       annotations:

Diff for: assets/thanos-ruler/thanos-ruler-prometheus-rule.yaml

@@ -5,12 +5,11 @@ metadata:
   namespace: openshift-user-workload-monitoring
 spec:
   groups:
-  - name: thanos-rule.rules
+  - name: thanos-rule
     rules:
     - alert: ThanosRuleQueueIsDroppingAlerts
       annotations:
-        description: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to queue
-          alerts.
+        description: Thanos Rule {{$labels.job}} is failing to queue alerts.
         summary: Thanos Rule is failing to queue alerts.
       expr: |
         sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job="thanos-ruler"}[5m])) > 0
@@ -19,8 +18,7 @@ spec:
         severity: critical
     - alert: ThanosRuleSenderIsFailingAlerts
       annotations:
-        description: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to send
-          alerts to alertmanager.
+        description: Thanos Rule {{$labels.job}} is failing to send alerts to alertmanager.
         summary: Thanos Rule is failing to send alerts to alertmanager.
       expr: |
         sum by (job) (rate(thanos_alert_sender_alerts_dropped_total{job="thanos-ruler"}[5m])) > 0
@@ -29,8 +27,7 @@ spec:
         severity: critical
     - alert: ThanosRuleHighRuleEvaluationFailures
       annotations:
-        description: Thanos Rule {{$labels.job}} {{$labels.pod}} is failing to evaluate
-          rules.
+        description: Thanos Rule {{$labels.job}} is failing to evaluate rules.
         summary: Thanos Rule is failing to evaluate rules.
       expr: |
         (
@@ -44,8 +41,7 @@ spec:
         severity: critical
     - alert: ThanosRuleHighRuleEvaluationWarnings
       annotations:
-        description: Thanos Rule {{$labels.job}} {{$labels.pod}} has high number of
-          evaluation warnings.
+        description: Thanos Rule {{$labels.job}} has high number of evaluation warnings.
         summary: Thanos Rule has high number of evaluation warnings.
       expr: |
         sum by (job) (rate(thanos_rule_evaluation_with_warnings_total{job="thanos-ruler"}[5m])) > 0
@@ -54,14 +50,14 @@ spec:
         severity: info
     - alert: ThanosRuleRuleEvaluationLatencyHigh
       annotations:
-        description: Thanos Rule {{$labels.job}}/{{$labels.pod}} has higher evaluation
+        description: Thanos Rule {{$labels.job}}/{{$labels.instance}} has higher evaluation
           latency than interval for {{$labels.rule_group}}.
         summary: Thanos Rule has high rule evaluation latency.
       expr: |
         (
-          sum by (job, pod, rule_group) (prometheus_rule_group_last_duration_seconds{job="thanos-ruler"})
+          sum by (job, instance, rule_group) (prometheus_rule_group_last_duration_seconds{job="thanos-ruler"})
         >
-          sum by (job, pod, rule_group) (prometheus_rule_group_interval_seconds{job="thanos-ruler"})
+          sum by (job, instance, rule_group) (prometheus_rule_group_interval_seconds{job="thanos-ruler"})
         )
       for: 5m
       labels:

Diff for: jsonnet/jsonnetfile.json

@@ -64,7 +64,7 @@
           "subdir": "mixin"
         }
       },
-      "version": "release-0.17"
+      "version": "95b2fe908b29977a20f695afac33f6257bb2d3ea"
     }
   ],
   "legacyImports": true

Diff for: jsonnet/jsonnetfile.lock.json

@@ -203,8 +203,8 @@
           "subdir": "mixin"
         }
       },
-      "version": "37e6ef61566c7c70793ba6d128f00c4c66cb2402",
-      "sum": "OptiWUMOHFrRGTZhSfxV1RCeXZ90qsefGNTD4lDYVG0="
+      "version": "95b2fe908b29977a20f695afac33f6257bb2d3ea",
+      "sum": "bE5eEPMulYMtz0lJSxDVnCXVTg/lqSSS5aEUUH3V19A="
     }
   ],
   "legacyImports": false

Diff for: jsonnet/patch-rules.libsonnet

@@ -74,6 +74,32 @@ local patchedRules = [
       },
     ],
   },
+  {
+    name: 'thanos-sidecar',
+    rules: [
+      {
+        alert: 'ThanosSidecarPrometheusDown',
+        'for': '1h',
+        labels: {
+          severity: 'warning',
+        },
+      },
+      {
+        alert: 'ThanosSidecarBucketOperationsFailed',
+        'for': '1h',
+        labels: {
+          severity: 'warning',
+        },
+      },
+      {
+        alert: 'ThanosSidecarUnhealthy',
+        'for': '1h',
+        labels: {
+          severity: 'warning',
+        },
+      },
+    ],
+  },
 ];
 
 local patchOrExcludeRule(rule, ruleSet, operation) =