Remove node-exporter alerts (#295)

Signed-off-by: ArthurSens <[email protected]>

Signed-off-by: ArthurSens <[email protected]>
Co-authored-by: Aleksandar Aleksandrov <[email protected]>

Author: ArthurSens

lib/alert-filter.libsonnet

@@ -68,6 +68,19 @@ local unwatedAlerts = [
   'KubeletDown',  // Re-added to platform-mixin
   'KubeClientErrors',
 
+  // From node-exporter
+  'NodeFilesystemAlmostOutOfSpace',  // Re-added to platform-mixin
+  'NodeFilesystemFilesFillingUp',
+  'NodeFilesystemAlmostOutOfFiles',  // Re-added to platform-mixin
+  'NodeNetworkReceiveErrs',
+  'NodeNetworkTransmitErrs',
+  'NodeTextFileCollectorScrapeError',
+  'NodeClockSkewDetected',
+  'NodeClockNotSynchronising',
+  'NodeRAIDDegraded',
+  'NodeRAIDDiskFailure',
+  'NodeFileDescriptorLimit',
+
   // From Alertmanager
   'AlertmanagerFailedReload',  // Re-added to platform-mixin
   'AlertmanagerFailedToSendAlerts',  // Re-added to platform-mixin

monitoring-satellite/manifests/kube-prometheus-rules/rules.yaml

@@ -764,199 +764,7 @@ spec:
         quantile: "0.5"
       record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
   - name: node-exporter
-    rules:
-    - alert: NodeFilesystemAlmostOutOfSpace
-      annotations:
-        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFilesystemAlmostOutOfSpace.md
-        summary: Filesystem has less than 5% space left.
-      expr: |
-        (
-          node_filesystem_avail_bytes{job="node-exporter",fstype!="shiftfs"} / node_filesystem_size_bytes{job="node-exporter",fstype!="shiftfs"} * 100 < 5
-        and
-          node_filesystem_readonly{job="node-exporter",fstype!="shiftfs"} == 0
-        )
-      for: 15m
-      labels:
-        severity: critical
-    - alert: NodeFilesystemAlmostOutOfSpace
-      annotations:
-        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFilesystemAlmostOutOfSpace.md
-        summary: Filesystem has less than 3% space left.
-      expr: |
-        (
-          node_filesystem_avail_bytes{job="node-exporter",fstype!="shiftfs"} / node_filesystem_size_bytes{job="node-exporter",fstype!="shiftfs"} * 100 < 3
-        and
-          node_filesystem_readonly{job="node-exporter",fstype!="shiftfs"} == 0
-        )
-      for: 15m
-      labels:
-        severity: critical
-    - alert: NodeFilesystemFilesFillingUp
-      annotations:
-        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFilesystemFilesFillingUp.md
-        summary: Filesystem is predicted to run out of inodes within the next 24 hours.
-      expr: |
-        (
-          node_filesystem_files_free{job="node-exporter",fstype!="shiftfs"} / node_filesystem_files{job="node-exporter",fstype!="shiftfs"} * 100 < 40
-        and
-          predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="shiftfs"}[6h], 24*60*60) < 0
-        and
-          node_filesystem_readonly{job="node-exporter",fstype!="shiftfs"} == 0
-        )
-      for: 1h
-      labels:
-        severity: warning
-    - alert: NodeFilesystemFilesFillingUp
-      annotations:
-        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFilesystemFilesFillingUp.md
-        summary: Filesystem is predicted to run out of inodes within the next 4 hours.
-      expr: |
-        (
-          node_filesystem_files_free{job="node-exporter",fstype!="shiftfs"} / node_filesystem_files{job="node-exporter",fstype!="shiftfs"} * 100 < 20
-        and
-          predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="shiftfs"}[6h], 4*60*60) < 0
-        and
-          node_filesystem_readonly{job="node-exporter",fstype!="shiftfs"} == 0
-        )
-      for: 1h
-      labels:
-        severity: critical
-    - alert: NodeFilesystemAlmostOutOfFiles
-      annotations:
-        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFilesystemAlmostOutOfFiles.md
-        summary: Filesystem has less than 5% inodes left.
-      expr: |
-        (
-          node_filesystem_files_free{job="node-exporter",fstype!="shiftfs"} / node_filesystem_files{job="node-exporter",fstype!="shiftfs"} * 100 < 5
-        and
-          node_filesystem_readonly{job="node-exporter",fstype!="shiftfs"} == 0
-        )
-      for: 1h
-      labels:
-        severity: warning
-    - alert: NodeFilesystemAlmostOutOfFiles
-      annotations:
-        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFilesystemAlmostOutOfFiles.md
-        summary: Filesystem has less than 3% inodes left.
-      expr: |
-        (
-          node_filesystem_files_free{job="node-exporter",fstype!="shiftfs"} / node_filesystem_files{job="node-exporter",fstype!="shiftfs"} * 100 < 3
-        and
-          node_filesystem_readonly{job="node-exporter",fstype!="shiftfs"} == 0
-        )
-      for: 1h
-      labels:
-        severity: critical
-    - alert: NodeNetworkReceiveErrs
-      annotations:
-        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeNetworkReceiveErrs.md
-        summary: Network interface is reporting many receive errors.
-      expr: |
-        rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
-      for: 1h
-      labels:
-        severity: warning
-    - alert: NodeNetworkTransmitErrs
-      annotations:
-        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeNetworkTransmitErrs.md
-        summary: Network interface is reporting many transmit errors.
-      expr: |
-        rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
-      for: 1h
-      labels:
-        severity: warning
-    - alert: NodeTextFileCollectorScrapeError
-      annotations:
-        description: Node Exporter text file collector failed to scrape.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeTextFileCollectorScrapeError.md
-        summary: Node Exporter text file collector failed to scrape.
-      expr: |
-        node_textfile_scrape_error{job="node-exporter"} == 1
-      labels:
-        severity: warning
-    - alert: NodeClockSkewDetected
-      annotations:
-        description: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeClockSkewDetected.md
-        summary: Clock skew detected.
-      expr: |
-        (
-          node_timex_offset_seconds{job="node-exporter"} > 0.05
-        and
-          deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
-        )
-        or
-        (
-          node_timex_offset_seconds{job="node-exporter"} < -0.05
-        and
-          deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
-        )
-      for: 10m
-      labels:
-        severity: warning
-    - alert: NodeClockNotSynchronising
-      annotations:
-        description: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeClockNotSynchronising.md
-        summary: Clock not synchronising.
-      expr: |
-        min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
-        and
-        node_timex_maxerror_seconds{job="node-exporter"} >= 16
-      for: 10m
-      labels:
-        severity: warning
-    - alert: NodeRAIDDegraded
-      annotations:
-        description: RAID array '{{ $labels.device }}' on {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeRAIDDegraded.md
-        summary: RAID Array is degraded
-      expr: |
-        node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"}) > 0
-      for: 15m
-      labels:
-        severity: critical
-    - alert: NodeRAIDDiskFailure
-      annotations:
-        description: At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeRAIDDiskFailure.md
-        summary: Failed device in RAID array
-      expr: |
-        node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)"} > 0
-      labels:
-        severity: warning
-    - alert: NodeFileDescriptorLimit
-      annotations:
-        description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFileDescriptorLimit.md
-        summary: Kernel is predicted to exhaust file descriptors limit soon.
-      expr: |
-        (
-          node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
-        )
-      for: 15m
-      labels:
-        severity: warning
-    - alert: NodeFileDescriptorLimit
-      annotations:
-        description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/NodeFileDescriptorLimit.md
-        summary: Kernel is predicted to exhaust file descriptors limit soon.
-      expr: |
-        (
-          node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
-        )
-      for: 15m
-      labels:
-        severity: critical
+    rules: []
   - name: node-exporter.rules
     rules:
     - expr: |