Skip to content

Commit 90bf787

Browse files
raptorsunraptorsun
committed
add limitScrapeBodySize to CMO ConfigMap
1 parent f0dad76 commit 90bf787

File tree

4 files changed

+95
-24
lines changed

4 files changed

+95
-24
lines changed

Diff for: ‎pkg/manifests/config.go

+48-21
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ import (
2929

3030
const (
3131
DefaultRetentionValue = "15d"
32+
33+
// Limit the body size from scrape queries
34+
// Assumptions: one node has maximum 110 pods, each pod exposes 100 metrics, each metric is expressed by at most 100 bytes.
35+
// 1.5x the size for a safe margin, it rounds up to 2MB.
36+
DefaultScrapeBodySizeLimit = "2MB"
3237
)
3338

3439
type Config struct {
@@ -105,6 +110,7 @@ type ClusterMonitoringConfiguration struct {
105110
K8sPrometheusAdapter *K8sPrometheusAdapter `json:"k8sPrometheusAdapter"`
106111
ThanosQuerierConfig *ThanosQuerierConfig `json:"thanosQuerier"`
107112
UserWorkloadEnabled *bool `json:"enableUserWorkload"`
113+
LimitScrapeBodySize *bool `json:"limitScrapeBodySize"`
108114
}
109115

110116
type Images struct {
@@ -168,16 +174,17 @@ type RemoteWriteSpec struct {
168174
}
169175

170176
type PrometheusK8sConfig struct {
171-
LogLevel string `json:"logLevel"`
172-
Retention string `json:"retention"`
173-
NodeSelector map[string]string `json:"nodeSelector"`
174-
Tolerations []v1.Toleration `json:"tolerations"`
175-
Resources *v1.ResourceRequirements `json:"resources"`
176-
ExternalLabels map[string]string `json:"externalLabels"`
177-
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
178-
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
179-
TelemetryMatches []string `json:"-"`
180-
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
177+
LogLevel string `json:"logLevel"`
178+
Retention string `json:"retention"`
179+
NodeSelector map[string]string `json:"nodeSelector"`
180+
Tolerations []v1.Toleration `json:"tolerations"`
181+
Resources *v1.ResourceRequirements `json:"resources"`
182+
ExternalLabels map[string]string `json:"externalLabels"`
183+
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
184+
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
185+
TelemetryMatches []string `json:"-"`
186+
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
187+
EnforcedBodySizeLimit string `json:"enforcedBodySizeLimit,omitempty"`
181188
}
182189

183190
type AdditionalAlertmanagerConfig struct {
@@ -384,6 +391,25 @@ func (c *Config) applyDefaults() {
384391
if c.ClusterMonitoringConfiguration.EtcdConfig == nil {
385392
c.ClusterMonitoringConfiguration.EtcdConfig = &EtcdConfig{}
386393
}
394+
if c.ClusterMonitoringConfiguration.LimitScrapeBodySize == nil {
395+
disable := false
396+
c.ClusterMonitoringConfiguration.LimitScrapeBodySize = &disable
397+
} else if *c.ClusterMonitoringConfiguration.LimitScrapeBodySize {
398+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = DefaultScrapeBodySizeLimit
399+
}
400+
}
401+
402+
func (c *Config) ActivatePromEnforceBodyLimit() {
403+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = DefaultScrapeBodySizeLimit
404+
if *c.ClusterMonitoringConfiguration.UserWorkloadEnabled {
405+
c.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit = DefaultScrapeBodySizeLimit
406+
}
407+
}
408+
func (c *Config) DeactivatePromEnforceBodyLimit() {
409+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = ""
410+
if *c.ClusterMonitoringConfiguration.UserWorkloadEnabled {
411+
c.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit = ""
412+
}
387413
}
388414

389415
func (c *Config) SetImages(images map[string]string) {
@@ -497,17 +523,18 @@ type UserWorkloadConfiguration struct {
497523
}
498524

499525
type PrometheusRestrictedConfig struct {
500-
LogLevel string `json:"logLevel"`
501-
Retention string `json:"retention"`
502-
NodeSelector map[string]string `json:"nodeSelector"`
503-
Tolerations []v1.Toleration `json:"tolerations"`
504-
Resources *v1.ResourceRequirements `json:"resources"`
505-
ExternalLabels map[string]string `json:"externalLabels"`
506-
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
507-
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
508-
EnforcedSampleLimit *uint64 `json:"enforcedSampleLimit"`
509-
EnforcedTargetLimit *uint64 `json:"enforcedTargetLimit"`
510-
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
526+
LogLevel string `json:"logLevel"`
527+
Retention string `json:"retention"`
528+
NodeSelector map[string]string `json:"nodeSelector"`
529+
Tolerations []v1.Toleration `json:"tolerations"`
530+
Resources *v1.ResourceRequirements `json:"resources"`
531+
ExternalLabels map[string]string `json:"externalLabels"`
532+
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
533+
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
534+
EnforcedSampleLimit *uint64 `json:"enforcedSampleLimit"`
535+
EnforcedTargetLimit *uint64 `json:"enforcedTargetLimit"`
536+
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
537+
EnforcedBodySizeLimit string `json:"enforcedBodySizeLimit,omitempty"`
511538
}
512539

513540
func (u *UserWorkloadConfiguration) applyDefaults() {

Diff for: ‎pkg/manifests/manifests.go

+8
Original file line numberDiff line numberDiff line change
@@ -1575,6 +1575,10 @@ func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret, trustedCABundle
15751575
p.Spec.Secrets = append(p.Spec.Secrets, getAdditionalAlertmanagerSecrets(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.AlertmanagerConfigs)...)
15761576
}
15771577

1578+
if f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit != "" {
1579+
p.Spec.EnforcedBodySizeLimit = f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit
1580+
}
1581+
15781582
return p, nil
15791583
}
15801584

@@ -1666,6 +1670,10 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
16661670
p.Spec.EnforcedTargetLimit = f.config.UserWorkloadConfiguration.Prometheus.EnforcedTargetLimit
16671671
}
16681672

1673+
if f.config.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit != "" {
1674+
p.Spec.EnforcedBodySizeLimit = f.config.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit
1675+
}
1676+
16691677
if f.config.Images.Thanos != "" {
16701678
p.Spec.Thanos.Image = &f.config.Images.Thanos
16711679
}

Diff for: ‎pkg/manifests/manifests_test.go

+33-3
Original file line numberDiff line numberDiff line change
@@ -1067,8 +1067,13 @@ ingress:
10671067
t.Fatal("Prometheus image is not configured correctly")
10681068
}
10691069

1070+
if p.Spec.EnforcedBodySizeLimit != "" {
1071+
t.Fatal("EnforcedBodySizeLimit is not set to empty by default")
1072+
}
1073+
10701074
kubeRbacProxyTLSCipherSuitesArg := ""
10711075
kubeRbacProxyMinTLSVersionArg := ""
1076+
10721077
for _, container := range p.Spec.Containers {
10731078
switch container.Name {
10741079
case "prometheus-proxy":
@@ -1170,6 +1175,31 @@ ingress:
11701175
}
11711176
}
11721177

1178+
func TestPrometheusK8sConfigurationBodySizeLimit(t *testing.T) {
1179+
c, err := NewConfigFromString(`
1180+
limitScrapeBodySize: true
1181+
`)
1182+
1183+
if err != nil {
1184+
t.Fatal(err)
1185+
}
1186+
1187+
f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c, defaultInfrastructureReader(), &fakeProxyReader{}, NewAssets(assetsPath), &APIServerConfig{})
1188+
p, err := f.PrometheusK8s(
1189+
"prometheus-k8s.openshift-monitoring.svc",
1190+
&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
1191+
&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
1192+
)
1193+
if err != nil {
1194+
t.Fatal(err)
1195+
}
1196+
1197+
if p.Spec.EnforcedBodySizeLimit != DefaultScrapeBodySizeLimit {
1198+
t.Fatalf("EnforcedBodySizeLimit is not configured correctly, expected %v, but got %v", DefaultScrapeBodySizeLimit, p.Spec.EnforcedBodySizeLimit)
1199+
}
1200+
1201+
}
1202+
11731203
func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
11741204
testCases := []struct {
11751205
name string
@@ -1224,7 +1254,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
12241254
config: `prometheusK8s:
12251255
additionalAlertmanagerConfigs:
12261256
- apiVersion: v2
1227-
scheme: https
1257+
scheme: https
12281258
bearerToken:
12291259
name: alertmanager1-bearer-token
12301260
key: token
@@ -1247,7 +1277,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
12471277
config: `prometheusK8s:
12481278
additionalAlertmanagerConfigs:
12491279
- apiVersion: v2
1250-
scheme: https
1280+
scheme: https
12511281
tlsConfig:
12521282
ca:
12531283
name: alertmanager-tls
@@ -1281,7 +1311,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
12811311
config: `prometheusK8s:
12821312
additionalAlertmanagerConfigs:
12831313
- apiVersion: v2
1284-
scheme: https
1314+
scheme: https
12851315
tlsConfig:
12861316
ca:
12871317
name: alertmanager-ca-tls

Diff for: ‎pkg/operator/operator.go

+6
Original file line numberDiff line numberDiff line change
@@ -785,6 +785,12 @@ func (o *Operator) Config(ctx context.Context, key string) (*manifests.Config, e
785785
}
786786
o.userWorkloadEnabled = *c.ClusterMonitoringConfiguration.UserWorkloadEnabled
787787

788+
if *c.ClusterMonitoringConfiguration.LimitScrapeBodySize {
789+
c.ActivatePromEnforceBodyLimit()
790+
} else {
791+
c.DeactivatePromEnforceBodyLimit()
792+
}
793+
788794
// Only fetch the token and cluster ID if they have not been specified in the config.
789795
if c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID == "" || c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token == "" {
790796
err := c.LoadClusterID(func() (*configv1.ClusterVersion, error) {

0 commit comments

Comments
 (0)