Skip to content

Commit b3aa278

Browse files
raptorsunraptorsun
committed
add limitScrapeBodySize to CMO ConfigMap
1 parent 5efe381 commit b3aa278

File tree

4 files changed

+97
-26
lines changed

4 files changed

+97
-26
lines changed

Diff for: ‎pkg/manifests/config.go

+50-23
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ import (
2929

3030
const (
3131
DefaultRetentionValue = "15d"
32+
33+
// Limit the body size from scrape queries
34+
// Assumptions: one node has maximum 110 pods, each pod exposes 100 metrics, each metric is expressed by at most 100 bytes.
35+
// 1.5x the size for a safe margin, it rounds up to 2MB.
36+
DefaultScrapeBodySizeLimit = "2MB"
3237
)
3338

3439
type Config struct {
@@ -105,6 +110,7 @@ type ClusterMonitoringConfiguration struct {
105110
K8sPrometheusAdapter *K8sPrometheusAdapter `json:"k8sPrometheusAdapter"`
106111
ThanosQuerierConfig *ThanosQuerierConfig `json:"thanosQuerier"`
107112
UserWorkloadEnabled *bool `json:"enableUserWorkload"`
113+
LimitScrapeBodySize *bool `json:"limitScrapeBodySize"`
108114
}
109115

110116
type Images struct {
@@ -168,17 +174,18 @@ type RemoteWriteSpec struct {
168174
}
169175

170176
type PrometheusK8sConfig struct {
171-
LogLevel string `json:"logLevel"`
172-
Retention string `json:"retention"`
173-
NodeSelector map[string]string `json:"nodeSelector"`
174-
Tolerations []v1.Toleration `json:"tolerations"`
175-
Resources *v1.ResourceRequirements `json:"resources"`
176-
ExternalLabels map[string]string `json:"externalLabels"`
177-
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
178-
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
179-
TelemetryMatches []string `json:"-"`
180-
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
181-
QueryLogFile string `json:"queryLogFile"`
177+
LogLevel string `json:"logLevel"`
178+
Retention string `json:"retention"`
179+
NodeSelector map[string]string `json:"nodeSelector"`
180+
Tolerations []v1.Toleration `json:"tolerations"`
181+
Resources *v1.ResourceRequirements `json:"resources"`
182+
ExternalLabels map[string]string `json:"externalLabels"`
183+
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
184+
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
185+
TelemetryMatches []string `json:"-"`
186+
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
187+
QueryLogFile string `json:"queryLogFile"`
188+
EnforcedBodySizeLimit string `json:"enforcedBodySizeLimit,omitempty"`
182189
}
183190

184191
type AdditionalAlertmanagerConfig struct {
@@ -385,6 +392,25 @@ func (c *Config) applyDefaults() {
385392
if c.ClusterMonitoringConfiguration.EtcdConfig == nil {
386393
c.ClusterMonitoringConfiguration.EtcdConfig = &EtcdConfig{}
387394
}
395+
if c.ClusterMonitoringConfiguration.LimitScrapeBodySize == nil {
396+
disable := false
397+
c.ClusterMonitoringConfiguration.LimitScrapeBodySize = &disable
398+
} else if *c.ClusterMonitoringConfiguration.LimitScrapeBodySize {
399+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = DefaultScrapeBodySizeLimit
400+
}
401+
}
402+
403+
func (c *Config) ActivatePromEnforceBodyLimit() {
404+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = DefaultScrapeBodySizeLimit
405+
if *c.ClusterMonitoringConfiguration.UserWorkloadEnabled {
406+
c.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit = DefaultScrapeBodySizeLimit
407+
}
408+
}
409+
func (c *Config) DeactivatePromEnforceBodyLimit() {
410+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = ""
411+
if *c.ClusterMonitoringConfiguration.UserWorkloadEnabled {
412+
c.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit = ""
413+
}
388414
}
389415

390416
func (c *Config) SetImages(images map[string]string) {
@@ -498,18 +524,19 @@ type UserWorkloadConfiguration struct {
498524
}
499525

500526
type PrometheusRestrictedConfig struct {
501-
LogLevel string `json:"logLevel"`
502-
Retention string `json:"retention"`
503-
NodeSelector map[string]string `json:"nodeSelector"`
504-
Tolerations []v1.Toleration `json:"tolerations"`
505-
Resources *v1.ResourceRequirements `json:"resources"`
506-
ExternalLabels map[string]string `json:"externalLabels"`
507-
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
508-
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
509-
EnforcedSampleLimit *uint64 `json:"enforcedSampleLimit"`
510-
EnforcedTargetLimit *uint64 `json:"enforcedTargetLimit"`
511-
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
512-
QueryLogFile string `json:"queryLogFile"`
527+
LogLevel string `json:"logLevel"`
528+
Retention string `json:"retention"`
529+
NodeSelector map[string]string `json:"nodeSelector"`
530+
Tolerations []v1.Toleration `json:"tolerations"`
531+
Resources *v1.ResourceRequirements `json:"resources"`
532+
ExternalLabels map[string]string `json:"externalLabels"`
533+
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
534+
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
535+
EnforcedSampleLimit *uint64 `json:"enforcedSampleLimit"`
536+
EnforcedTargetLimit *uint64 `json:"enforcedTargetLimit"`
537+
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
538+
QueryLogFile string `json:"queryLogFile"`
539+
EnforcedBodySizeLimit string `json:"enforcedBodySizeLimit,omitempty"`
513540
}
514541

515542
func (u *UserWorkloadConfiguration) applyDefaults() {

Diff for: ‎pkg/manifests/manifests.go

+8
Original file line numberDiff line numberDiff line change
@@ -1582,6 +1582,10 @@ func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret, trustedCABundle
15821582
p.Spec.Secrets = append(p.Spec.Secrets, getAdditionalAlertmanagerSecrets(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.AlertmanagerConfigs)...)
15831583
}
15841584

1585+
if f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit != "" {
1586+
p.Spec.EnforcedBodySizeLimit = f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit
1587+
}
1588+
15851589
return p, nil
15861590
}
15871591

@@ -1673,6 +1677,10 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
16731677
p.Spec.EnforcedTargetLimit = f.config.UserWorkloadConfiguration.Prometheus.EnforcedTargetLimit
16741678
}
16751679

1680+
if f.config.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit != "" {
1681+
p.Spec.EnforcedBodySizeLimit = f.config.UserWorkloadConfiguration.Prometheus.EnforcedBodySizeLimit
1682+
}
1683+
16761684
if f.config.Images.Thanos != "" {
16771685
p.Spec.Thanos.Image = &f.config.Images.Thanos
16781686
}

Diff for: ‎pkg/manifests/manifests_test.go

+33-3
Original file line numberDiff line numberDiff line change
@@ -1068,8 +1068,13 @@ ingress:
10681068
t.Fatal("Prometheus image is not configured correctly")
10691069
}
10701070

1071+
if p.Spec.EnforcedBodySizeLimit != "" {
1072+
t.Fatal("EnforcedBodySizeLimit is not set to empty by default")
1073+
}
1074+
10711075
kubeRbacProxyTLSCipherSuitesArg := ""
10721076
kubeRbacProxyMinTLSVersionArg := ""
1077+
10731078
for _, container := range p.Spec.Containers {
10741079
switch container.Name {
10751080
case "prometheus-proxy":
@@ -1175,6 +1180,31 @@ ingress:
11751180
}
11761181
}
11771182

1183+
func TestPrometheusK8sConfigurationBodySizeLimit(t *testing.T) {
1184+
c, err := NewConfigFromString(`
1185+
limitScrapeBodySize: true
1186+
`)
1187+
1188+
if err != nil {
1189+
t.Fatal(err)
1190+
}
1191+
1192+
f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c, defaultInfrastructureReader(), &fakeProxyReader{}, NewAssets(assetsPath), &APIServerConfig{})
1193+
p, err := f.PrometheusK8s(
1194+
"prometheus-k8s.openshift-monitoring.svc",
1195+
&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
1196+
&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
1197+
)
1198+
if err != nil {
1199+
t.Fatal(err)
1200+
}
1201+
1202+
if p.Spec.EnforcedBodySizeLimit != DefaultScrapeBodySizeLimit {
1203+
t.Fatalf("EnforcedBodySizeLimit is not configured correctly, expected %v, but got %v", DefaultScrapeBodySizeLimit, p.Spec.EnforcedBodySizeLimit)
1204+
}
1205+
1206+
}
1207+
11781208
func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
11791209
testCases := []struct {
11801210
name string
@@ -1229,7 +1259,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
12291259
config: `prometheusK8s:
12301260
additionalAlertmanagerConfigs:
12311261
- apiVersion: v2
1232-
scheme: https
1262+
scheme: https
12331263
bearerToken:
12341264
name: alertmanager1-bearer-token
12351265
key: token
@@ -1252,7 +1282,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
12521282
config: `prometheusK8s:
12531283
additionalAlertmanagerConfigs:
12541284
- apiVersion: v2
1255-
scheme: https
1285+
scheme: https
12561286
tlsConfig:
12571287
ca:
12581288
name: alertmanager-tls
@@ -1286,7 +1316,7 @@ func TestPrometheusK8sAdditionalAlertManagerConfigsSecret(t *testing.T) {
12861316
config: `prometheusK8s:
12871317
additionalAlertmanagerConfigs:
12881318
- apiVersion: v2
1289-
scheme: https
1319+
scheme: https
12901320
tlsConfig:
12911321
ca:
12921322
name: alertmanager-ca-tls

Diff for: ‎pkg/operator/operator.go

+6
Original file line numberDiff line numberDiff line change
@@ -785,6 +785,12 @@ func (o *Operator) Config(ctx context.Context, key string) (*manifests.Config, e
785785
}
786786
o.userWorkloadEnabled = *c.ClusterMonitoringConfiguration.UserWorkloadEnabled
787787

788+
if *c.ClusterMonitoringConfiguration.LimitScrapeBodySize {
789+
c.ActivatePromEnforceBodyLimit()
790+
} else {
791+
c.DeactivatePromEnforceBodyLimit()
792+
}
793+
788794
// Only fetch the token and cluster ID if they have not been specified in the config.
789795
if c.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID == "" || c.ClusterMonitoringConfiguration.TelemeterClientConfig.Token == "" {
790796
err := c.LoadClusterID(func() (*configv1.ClusterVersion, error) {

0 commit comments

Comments
 (0)