Skip to content

Commit 56f5fc2

Browse files
tremestremes
committed
remove firing alerts from the config/metrics file
1 parent f7c5ffd commit 56f5fc2

File tree

4 files changed

+4
-58
lines changed

4 files changed

+4
-58
lines changed

‎docs/gathered-data.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1220,7 +1220,6 @@ Gathered metrics:
12201220
- `console_helm_uninstalls_total`
12211221
- `etcd_server_slow_apply_total`
12221222
- `etcd_server_slow_read_indexes_total`
1223-
- followed by at most 1000 lines of `ALERTS` metric
12241223

12251224
### API Reference
12261225
None
@@ -1254,6 +1253,7 @@ None
12541253
- `openshift_apps_deploymentconfigs_strategy_total` introduced in version 4.13+ and backported to 4.12.5+ version
12551254
- `etcd_server_slow_apply_total` introduced in version 4.16+
12561255
- `etcd_server_slow_read_indexes_total` introduced in version 4.16+
1256+
- `ALERTS` removed in version 4.17+
12571257

12581258

12591259
## MutatingWebhookConfigurations

‎docs/insights-archive-sample/config/metrics

+1-17
Original file line numberDiff line numberDiff line change
@@ -129,20 +129,4 @@ virt_platform{container="kube-rbac-proxy",endpoint="https",instance="ci-ln-k19wb
129129
virt_platform{container="kube-rbac-proxy",endpoint="https",instance="ci-ln-k19wbxk-f76d1-6qdmf-worker-b-94cjz",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-dmnrm",service="node-exporter",type="gcp",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1620977245743
130130
virt_platform{container="kube-rbac-proxy",endpoint="https",instance="ci-ln-k19wbxk-f76d1-6qdmf-worker-b-94cjz",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-dmnrm",service="node-exporter",type="kvm",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1620977245743
131131
virt_platform{container="kube-rbac-proxy",endpoint="https",instance="ci-ln-k19wbxk-f76d1-6qdmf-master-0",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-2dsf2",service="node-exporter",type="gcp",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1620977246807
132-
virt_platform{container="kube-rbac-proxy",endpoint="https",instance="ci-ln-k19wbxk-f76d1-6qdmf-worker-c-44ttf",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-56fn6",service="node-exporter",type="gcp",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1620977240435
133-
# ALERTS 16/1000
134-
# TYPE ALERTS untyped
135-
ALERTS{alertname="Watchdog",alertstate="firing",severity="none",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793310163
136-
ALERTS{alertname="KubeMemoryOvercommit",alertstate="firing",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793311534
137-
ALERTS{alertname="AlertmanagerReceiversNotConfigured",alertstate="firing",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793316662
138-
ALERTS{alertname="KubePodNotReady",alertstate="firing",namespace="openshift-etcd",pod="etcd-quorum-guard-587fd6c776-xg5zw",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793320590
139-
ALERTS{alertname="KubePodNotReady",alertstate="firing",namespace="openshift-etcd",pod="etcd-quorum-guard-587fd6c776-czv8b",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793320590
140-
ALERTS{alertname="KubePodNotReady",alertstate="firing",namespace="openshift-ingress",pod="router-default-6f59db78db-gh48w",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793320590
141-
ALERTS{alertname="KubeDeploymentReplicasMismatch",alertstate="firing",container="kube-rbac-proxy-main",deployment="router-default",endpoint="https-main",instance="10.129.0.12:8443",job="kube-state-metrics",namespace="openshift-ingress",pod="kube-state-metrics-664f855c7f-9vbzh",service="kube-state-metrics",severity="warning",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793320590
142-
ALERTS{alertname="KubeDeploymentReplicasMismatch",alertstate="firing",container="kube-rbac-proxy-main",deployment="etcd-quorum-guard",endpoint="https-main",instance="10.129.0.12:8443",job="kube-state-metrics",namespace="openshift-etcd",pod="kube-state-metrics-664f855c7f-9vbzh",service="kube-state-metrics",severity="warning",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793320590
143-
ALERTS{alertname="etcdHighCommitDurations",alertstate="firing",endpoint="etcd-metrics",instance="10.10.94.191:9979",job="etcd",namespace="openshift-etcd",pod="etcd-master-0.tremes.lab.rdu2.cee.redhat.com",service="etcd",severity="warning",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793295353
144-
ALERTS{alertname="ClusterOperatorDown",alertstate="firing",endpoint="metrics",instance="10.10.94.191:9099",job="cluster-version-operator",name="ingress",namespace="openshift-cluster-version",pod="cluster-version-operator-644d79c75d-xl7z5",service="cluster-version-operator",severity="critical",version="4.6.15",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793309213
145-
ALERTS{alertname="ClusterOperatorDegraded",alertstate="firing",condition="Degraded",endpoint="metrics",instance="10.10.94.191:9099",job="cluster-version-operator",name="ingress",namespace="openshift-cluster-version",pod="cluster-version-operator-644d79c75d-xl7z5",reason="IngressControllersDegraded",service="cluster-version-operator",severity="critical",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793309213
146-
ALERTS{alertname="KubeAPIErrorBudgetBurn",alertstate="pending",long="3d",severity="warning",short="6h",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793298512
147-
ALERTS{alertname="ClusterNotUpgradeable",alertstate="firing",condition="Upgradeable",endpoint="metrics",name="version",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"} 1 1612793309213
148-
ALERTS{alertname="KubePodNotReady",alertstate="pending",namespace="openshift-insights",pod="insights-operator-f7df674b4-x9qtw",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-1"}
132+
virt_platform{container="kube-rbac-proxy",endpoint="https",instance="ci-ln-k19wbxk-f76d1-6qdmf-worker-c-44ttf",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-56fn6",service="node-exporter",type="gcp",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1620977240435

‎pkg/gatherers/clusterconfig/gather_most_recent_metrics.go

+1-38
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,14 @@ package clusterconfig
22

33
import (
44
"context"
5-
"fmt"
6-
"io"
75

86
"k8s.io/client-go/rest"
97
"k8s.io/klog/v2"
108

119
"github.com/openshift/insights-operator/pkg/record"
12-
"github.com/openshift/insights-operator/pkg/utils"
1310
"github.com/openshift/insights-operator/pkg/utils/marshal"
1411
)
1512

16-
const (
17-
// metricsAlertsLinesLimit is the maximal number of lines read from monitoring Prometheus
18-
// 500 KiB of alerts is limit, one alert line has typically 450 bytes => 1137 lines.
19-
// This number has been rounded to 1000 for simplicity.
20-
// Formerly, the `500 * 1024 / 450` expression was used instead.
21-
metricsAlertsLinesLimit = 1000
22-
)
23-
2413
// GatherMostRecentMetrics Collects cluster Federated Monitoring metrics.
2514
//
2615
// The GET REST query to URL /federate
@@ -34,7 +23,6 @@ const (
3423
// - `console_helm_uninstalls_total`
3524
// - `etcd_server_slow_apply_total`
3625
// - `etcd_server_slow_read_indexes_total`
37-
// - followed by at most 1000 lines of `ALERTS` metric
3826
//
3927
// ### API Reference
4028
// None
@@ -68,6 +56,7 @@ const (
6856
// - `openshift_apps_deploymentconfigs_strategy_total` introduced in version 4.13+ and backported to 4.12.5+ version
6957
// - `etcd_server_slow_apply_total` introduced in version 4.16+
7058
// - `etcd_server_slow_read_indexes_total` introduced in version 4.16+
59+
// - `ALERTS` removed in version 4.17+
7160
func (g *Gatherer) GatherMostRecentMetrics(ctx context.Context) ([]record.Record, []error) {
7261
metricsRESTClient, err := rest.RESTClientFor(g.metricsGatherKubeConfig)
7362
if err != nil {
@@ -97,32 +86,6 @@ func gatherMostRecentMetrics(ctx context.Context, metricsClient rest.Interface)
9786
return nil, []error{err}
9887
}
9988

100-
rsp, err := metricsClient.Get().AbsPath("federate").
101-
Param("match[]", "ALERTS").
102-
Stream(ctx)
103-
if err != nil {
104-
klog.Errorf("Unable to retrieve most recent alerts from metrics: %v", err)
105-
return nil, []error{err}
106-
}
107-
r := utils.NewLineLimitReader(rsp, metricsAlertsLinesLimit)
108-
alerts, err := io.ReadAll(r)
109-
if err != nil && err != io.EOF {
110-
klog.Errorf("Unable to read most recent alerts from metrics: %v", err)
111-
return nil, []error{err}
112-
}
113-
114-
remainingAlertLines, err := utils.CountLines(rsp)
115-
if err != nil && err != io.EOF {
116-
klog.Errorf("Unable to count truncated lines of alerts metric: %v", err)
117-
return nil, []error{err}
118-
}
119-
totalAlertCount := r.GetTotalLinesRead() + remainingAlertLines
120-
121-
// # ALERTS <Total Alerts Lines>/<Alerts Line Limit>
122-
// The total number of alerts will typically be greater than the true number of alerts by 2
123-
// because the `# TYPE ALERTS untyped` header and the final empty line are counter in.
124-
data = append(data, []byte(fmt.Sprintf("# ALERTS %d/%d\n", totalAlertCount, metricsAlertsLinesLimit))...)
125-
data = append(data, alerts...)
12689
records := []record.Record{
12790
{Name: "config/metrics", Item: marshal.RawByte(data), AlwaysStored: true},
12891
}

‎pkg/gatherers/clusterconfig/gather_most_recent_metrics_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,8 @@ func Test_gatherMostRecentMetrics(t *testing.T) {
5555
wantRecords: []record.Record{
5656
{
5757
Name: "config/metrics",
58-
Item: marshal.RawByte(`test# ALERTS 1/1000
59-
test`),
6058
AlwaysStored: true,
59+
Item: marshal.RawByte(`test`),
6160
},
6261
},
6362
wantErrors: nil,

0 commit comments

Comments
 (0)