add alert for scrape body size exceeding limit

Author: raptorsun

assets/prometheus-k8s/prometheus-rule.yaml

@@ -245,3 +245,14 @@ spec:
       for: 5m
       labels:
         severity: critical
+    - alert: PrometheusScrapeBodySizeLimitHit
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped
+          {{ printf "%.0f" $value }} targets because some scraped targets have exceeded
+          the body size limit.
+        summary: Prometheus has dropped targets because some scraped targets have
+          exceeded the body size limit.
+      expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])>0
+      for: 15m
+      labels:
+        severity: warning

jsonnet/utils/sanitize-rules.libsonnet

@@ -326,6 +326,26 @@ local patchedRules = [
   },
 ];
 
+local addedRules = [
+  {
+    name: 'prometheus',
+    rules: [
+      {
+        alert: 'PrometheusScrapeBodySizeLimitHit',
+        expr: 'increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])>0',
+        'for': '15m',
+        labels: {
+          severity: 'warning',
+        },
+        annotations: {
+          description: 'Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some scraped targets have exceeded the body size limit.',
+          summary: 'Prometheus has dropped targets because some scraped targets have exceeded the body size limit.',
+        },
+      },
+    ],
+  },
+];
+
 local openShiftRunbook(runbook) =
   'https://github.com/openshift/runbooks/blob/master/' + runbook;
 
@@ -389,7 +409,10 @@ local patchOrExcludeRuleGroup(group, groupSet, operation) =
   if std.length(groupSet) == 0 then
     [group.rules]
   else if (group.name == groupSet[0].name) then
-    [patchOrExcludeRule(rule, groupSet[0].rules, operation) for rule in group.rules]
+    if operation == 'add' then
+      [group.rules + groupSet[0].rules]
+    else
+      [patchOrExcludeRule(rule, groupSet[0].rules, operation) for rule in group.rules]
   else
     [] + patchOrExcludeRuleGroup(group, groupSet[1:], operation);
 
@@ -423,6 +446,20 @@ local patchOrExcludeRuleGroup(group, groupSet, operation) =
     },
   },
 
+  addRule(o): o {
+    [if (o.kind == 'PrometheusRule') then 'spec']+: {
+      groups: std.map(
+        function(group)
+          group {
+            rules: std.flattenArrays(
+              patchOrExcludeRuleGroup(group, addedRules, 'add')
+            ),
+          },
+        super.groups,
+      ),
+    },
+  },
+
   addRunbookUrls(o): o {
     [if (o.kind == 'PrometheusRule') then 'spec']+: k8sMixinUtils.mapRuleGroups(addRunbookUrl),
   },
@@ -450,9 +487,14 @@ local patchOrExcludeRuleGroup(group, groupSet, operation) =
     for k in std.objectFields(o)
   },
 
+  addRules(o): {
+    [k]: $.addRule(o[k])
+    for k in std.objectFields(o)
+  },
+
   // shorthand for rule patching, rule excluding, and runbook_url removal
   sanitizeAlertRules(o): {
-    [k]: $.addRunbookUrls($.removeRunbookUrls($.patchRule($.excludeRule(o[k]))))
+    [k]: $.addRunbookUrls($.removeRunbookUrls($.addRule($.patchRule($.excludeRule(o[k])))))
     for k in std.objectFields(o)
   },
 }