jsonnet: adjust thanos-sidecar alerts

Decrease severity of all the Thanos sidecar alerts to warning and
increase their duration to 1h.

Signed-off-by: Damien Grisonnet <[email protected]>

Author: dgrisonnet

assets/prometheus-k8s/prometheus-rule.yaml

@@ -248,25 +248,26 @@ spec:
         summary: Thanos Sidecar cannot connect to Prometheus
       expr: |
         sum by (job, instance) (thanos_sidecar_prometheus_up{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"} == 0)
-      for: 5m
+      for: 1h
       labels:
-        severity: critical
+        severity: warning
     - alert: ThanosSidecarBucketOperationsFailed
       annotations:
         description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} bucket operations
           are failing
         summary: Thanos Sidecar bucket operations are failing
       expr: |
         rate(thanos_objstore_bucket_operation_failures_total{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"}[5m]) > 0
-      for: 5m
+      for: 1h
       labels:
-        severity: critical
+        severity: warning
     - alert: ThanosSidecarUnhealthy
       annotations:
         description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for
           more than {{ $value }} seconds.
         summary: Thanos Sidecar is unhealthy.
       expr: |
         time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"prometheus-(k8s|user-workload)-thanos-sidecar"})) by (job,pod) >= 240
+      for: 1h
       labels:
-        severity: critical
+        severity: warning

jsonnet/patch-rules.libsonnet

@@ -74,6 +74,32 @@ local patchedRules = [
       },
     ],
   },
+  {
+    name: 'thanos-sidecar',
+    rules: [
+      {
+        alert: 'ThanosSidecarPrometheusDown',
+        'for': '1h',
+        labels: {
+          severity: 'warning',
+        },
+      },
+      {
+        alert: 'ThanosSidecarBucketOperationsFailed',
+        'for': '1h',
+        labels: {
+          severity: 'warning',
+        },
+      },
+      {
+        alert: 'ThanosSidecarUnhealthy',
+        'for': '1h',
+        labels: {
+          severity: 'warning',
+        },
+      },
+    ],
+  },
 ];
 
 local patchOrExcludeRule(rule, ruleSet, operation) =