Merge pull request #1087 from simonpasquier/bz1934163

Bug 1934163: adjust Thanos querier alerting rules

Author: openshift-merge-robot

Diff for: CHANGELOG.md

@@ -1,5 +1,13 @@
 # Note: This CHANGELOG is only for the monitoring team to track all monitoring related changes. Please see OpenShift release notes for official changes.
 
+## 4.8
+
+- [#1087](https://github.com/openshift/cluster-monitoring-operator/pull/1087) Decrease alert severity to "warning" for ThanosQueryHttpRequestQueryErrorRateHigh and ThanosQueryHttpRequestQueryRangeErrorRateHigh alerts.
+- [#1087](https://github.com/openshift/cluster-monitoring-operator/pull/1087) Increase "for" duration to 1 hour for all Thanos query alerts.
+- [#1087](https://github.com/openshift/cluster-monitoring-operator/pull/1087) Remove ThanosQueryInstantLatencyHigh and ThanosQueryRangeLatencyHigh alerts.
+- [#1090](https://github.com/openshift/cluster-monitoring-operator/pull/1090) Decrease alert severity to "warning" for all Thanos sidecar alerts.
+- [#1090](https://github.com/openshift/cluster-monitoring-operator/pull/1090) Increase "for" duration to 1 hour for all Thanos sidecar alerts.
+
 ## 4.7
 
 - [#963](https://github.com/openshift/cluster-monitoring-operator/pull/963) bump mixins to include new etcd alerts

Diff for: assets/thanos-querier/prometheus-rule.yaml

@@ -23,9 +23,9 @@ spec:
         /
           sum(rate(http_requests_total{job="thanos-querier", handler="query"}[5m]))
         ) * 100 > 5
-      for: 5m
+      for: 1h
       labels:
-        severity: critical
+        severity: warning
     - alert: ThanosQueryHttpRequestQueryRangeErrorRateHigh
       annotations:
         description: Thanos Query {{$labels.job}} is failing to handle {{ $value |
@@ -37,9 +37,9 @@ spec:
         /
           sum(rate(http_requests_total{job="thanos-querier", handler="query_range"}[5m]))
         ) * 100 > 5
-      for: 5m
+      for: 1h
       labels:
-        severity: critical
+        severity: warning
     - alert: ThanosQueryGrpcServerErrorRate
       annotations:
         description: Thanos Query {{$labels.job}} is failing to handle {{ $value |
@@ -52,7 +52,7 @@ spec:
           sum by (job) (rate(grpc_server_started_total{job="thanos-querier"}[5m]))
         * 100 > 5
         )
-      for: 5m
+      for: 1h
       labels:
         severity: warning
     - alert: ThanosQueryGrpcClientErrorRate
@@ -66,7 +66,7 @@ spec:
         /
           sum by (job) (rate(grpc_client_started_total{job="thanos-querier"}[5m]))
         ) * 100 > 5
-      for: 5m
+      for: 1h
       labels:
         severity: warning
     - alert: ThanosQueryHighDNSFailures
@@ -80,34 +80,6 @@ spec:
         /
           sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job="thanos-querier"}[5m]))
         ) * 100 > 1
-      for: 15m
+      for: 1h
       labels:
         severity: warning
-    - alert: ThanosQueryInstantLatencyHigh
-      annotations:
-        description: Thanos Query {{$labels.job}} has a 99th percentile latency of
-          {{ $value }} seconds for instant queries.
-        summary: Thanos Query has high latency for queries.
-      expr: |
-        (
-          histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job="thanos-querier", handler="query"}[5m]))) > 40
-        and
-          sum by (job) (rate(http_request_duration_seconds_bucket{job="thanos-querier", handler="query"}[5m])) > 0
-        )
-      for: 10m
-      labels:
-        severity: critical
-    - alert: ThanosQueryRangeLatencyHigh
-      annotations:
-        description: Thanos Query {{$labels.job}} has a 99th percentile latency of
-          {{ $value }} seconds for range queries.
-        summary: Thanos Query has high latency for queries.
-      expr: |
-        (
-          histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job="thanos-querier", handler="query_range"}[5m]))) > 90
-        and
-          sum by (job) (rate(http_request_duration_seconds_count{job="thanos-querier", handler="query_range"}[5m])) > 0
-        )
-      for: 10m
-      labels:
-        severity: critical

Diff for: jsonnet/patch-rules.libsonnet

@@ -58,6 +58,13 @@ local excludedRules = [
       { alert: 'KubeDeploymentReplicasMismatch' },
     ],
   },
+  {
+    name: 'thanos-query',
+    rules: [
+      { alert: 'ThanosQueryInstantLatencyHigh' },
+      { alert: 'ThanosQueryRangeLatencyHigh' },
+    ],
+  },
 ];
 
 local patchedRules = [
@@ -78,21 +85,19 @@ local patchedRules = [
     name: 'thanos-sidecar',
     rules: [
       {
-        alert: 'ThanosSidecarPrometheusDown',
-        'for': '1h',
-        labels: {
-          severity: 'warning',
-        },
-      },
-      {
-        alert: 'ThanosSidecarBucketOperationsFailed',
+        alert: '',
         'for': '1h',
         labels: {
           severity: 'warning',
         },
       },
+    ],
+  },
+  {
+    name: 'thanos-query',
+    rules: [
       {
-        alert: 'ThanosSidecarUnhealthy',
+        alert: '',
         'for': '1h',
         labels: {
           severity: 'warning',
@@ -108,7 +113,12 @@ local patchOrExcludeRule(rule, ruleSet, operation) =
   else if (('alert' in rule && 'alert' in ruleSet[0]) && std.startsWith(rule.alert, ruleSet[0].alert)) ||
           (('record' in rule && 'record' in ruleSet[0]) && std.startsWith(rule.record, ruleSet[0].record)) then
     if operation == 'patch' then
-      [std.mergePatch(rule, ruleSet[0])]
+      local patch = {
+        [k]: ruleSet[0][k]
+        for k in std.objectFields(ruleSet[0])
+        if k != 'alert' && k != 'record'
+      };
+      [std.mergePatch(rule, patch)]
     else
       []
   else
@@ -123,6 +133,9 @@ local patchOrExcludeRuleGroup(group, groupSet, operation) =
     [] + patchOrExcludeRuleGroup(group, groupSet[1:], operation);
 
 {
+  // excludedRules removes upstream rules that we don't want to carry in CMO.
+  // It can remove specific rules from a rules group (see excludedRules) or a
+  // whole rules group (see excludedRuleGroups).
   excludeRules(o): {
     local exclude(o) = o {
       [if (o.kind == 'PrometheusRule') then 'spec']+: {
@@ -142,6 +155,12 @@ local patchOrExcludeRuleGroup(group, groupSet, operation) =
     for k in std.objectFields(o)
   },
 
+  // patchRules adapts upstream rules to comply with OpenShift requirements
+  // (such as extending the for duration, changing alert severity, and so on).
+  // The patches are defined in the patchedRules array where each item contains
+  // the name of the affected group and the list of patches keyed by their
+  // 'alert' or 'record' identifier. The function will apply the patch to every
+  // alerting/recording rule in the group whose name starts by the identifier.
   patchRules(o): {
     local patch(o) = o {
       [if (o.kind == 'PrometheusRule') then 'spec']+: {