jsonnet: remove etcd gRPC calls failed alerts

backport openshift#340

Author: paulfantom

Diff for: assets/prometheus-k8s/rules.yaml

@@ -1113,30 +1113,6 @@ spec:
       for: 15m
       labels:
         severity: warning
-    - alert: EtcdHighNumberOfFailedGRPCRequests
-      annotations:
-        message: 'Etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for
-          {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
-      expr: |
-        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
-          /
-        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
-          > 1
-      for: 10m
-      labels:
-        severity: warning
-    - alert: EtcdHighNumberOfFailedGRPCRequests
-      annotations:
-        message: 'Etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for
-          {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
-      expr: |
-        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
-          /
-        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
-          > 5
-      for: 5m
-      labels:
-        severity: critical
     - alert: EtcdGRPCRequestsSlow
       annotations:
         message: 'Etcd cluster "{{ $labels.job }}": gRPC requests to {{ $labels.grpc_method

Diff for: jsonnet/main.jsonnet

@@ -156,6 +156,20 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
         rule
     );
     utils.mapRuleGroups(replaceKubePodCrashLoopingExpression),
+} + {
+  // Remove constantly firing etcd gRPC alerts.
+  // https://bugzilla.redhat.com/show_bug.cgi?id=1717398
+  prometheusAlerts+:: {
+    groups:
+      std.map(
+        function(ruleGroup)
+          if ruleGroup.name == 'etcd' then
+            ruleGroup { rules: std.filter(function(rule) !('alert' in rule && rule.alert == 'EtcdHighNumberOfFailedGRPCRequests'), ruleGroup.rules) }
+          else
+            ruleGroup,
+        super.groups,
+      ),
+  },
 };
 
 { ['prometheus-operator/' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +