Skip to content

Commit edf383b

Browse files
raptorsunraptorsun
committed
add prometheusK8s.enforcedBodySizeLimit to CMO ConfigMap, limiting the
bodysize when scraping metric. Empty value or 0 means bodysize limit. "automatic" for automatically deduced bodysize limit.
1 parent c97c6f7 commit edf383b

File tree

9 files changed

+332
-17
lines changed

9 files changed

+332
-17
lines changed

Diff for: ‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
- [#1601](https://github.com/openshift/cluster-monitoring-operator/pull/1601) Expose the /federate endpoint of UWM Prometheus as a service
1010
- [#1617](https://github.com/openshift/cluster-monitoring-operator/pull/1617) Add Oauth2 setting to PrometheusK8s remoteWrite config
1111
- [#1598](https://github.com/openshift/cluster-monitoring-operator/pull/1598) Expose Authorization settings for remote write in the CMO configuration
12+
- [#1467](https://github.com/openshift/cluster-monitoring-operator/pull/1467) Add bodysize limit for metric scraping
1213

1314
## 4.10
1415

Diff for: ‎pkg/client/client.go

+18
Original file line numberDiff line numberDiff line change
@@ -1524,6 +1524,24 @@ func (c *Client) DeleteRole(ctx context.Context, role *rbacv1.Role) error {
15241524
return err
15251525
}
15261526

1527+
func (c *Client) PodCapacity(ctx context.Context) (int, error) {
1528+
nodes, err := c.kclient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
1529+
if err != nil {
1530+
return 0, err
1531+
}
1532+
var podCapacityTotal int64
1533+
for _, node := range nodes.Items {
1534+
podsCount, succeeded := node.Status.Capacity.Pods().AsInt64()
1535+
if !succeeded {
1536+
klog.Warningf("Cannot get pod capacity from node: %s. Error: %v", node.Name, err)
1537+
continue
1538+
}
1539+
podCapacityTotal += podsCount
1540+
}
1541+
1542+
return int(podCapacityTotal), nil
1543+
}
1544+
15271545
// mergeMetadata merges labels and annotations from `existing` map into `required` one where `required` has precedence
15281546
// over `existing` keys and values. Additionally function performs filtering of labels and annotations from `exiting` map
15291547
// where keys starting from string defined in `metadataPrefix` are deleted. This prevents issues with preserving stale

Diff for: ‎pkg/client/client_test.go

+48
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
appsv1 "k8s.io/api/apps/v1"
2626
v1 "k8s.io/api/core/v1"
2727
rbacv1 "k8s.io/api/rbac/v1"
28+
"k8s.io/apimachinery/pkg/api/resource"
2829
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2930

3031
monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -1836,3 +1837,50 @@ func TestCreateOrUpdateValidatingWebhookConfiguration(t *testing.T) {
18361837
})
18371838
}
18381839
}
1840+
1841+
func TestPodCapacity(t *testing.T) {
1842+
ctx := context.Background()
1843+
node1 := v1.Node{
1844+
ObjectMeta: metav1.ObjectMeta{
1845+
Name: "node1",
1846+
},
1847+
Status: v1.NodeStatus{
1848+
Capacity: v1.ResourceList{
1849+
v1.ResourcePods: resource.MustParse("100"),
1850+
},
1851+
},
1852+
}
1853+
node2 := v1.Node{
1854+
ObjectMeta: metav1.ObjectMeta{
1855+
Name: "node2",
1856+
},
1857+
Status: v1.NodeStatus{
1858+
Capacity: v1.ResourceList{
1859+
v1.ResourcePods: resource.MustParse("50"),
1860+
},
1861+
},
1862+
}
1863+
nodeList := v1.NodeList{
1864+
Items: []v1.Node{
1865+
node1,
1866+
node2,
1867+
},
1868+
}
1869+
t.Run("sum 2 nodes pod capacity", func(st *testing.T) {
1870+
1871+
c := Client{
1872+
kclient: fake.NewSimpleClientset(nodeList.DeepCopy()),
1873+
}
1874+
1875+
podCapacity, err := c.PodCapacity(ctx)
1876+
1877+
if err != nil {
1878+
t.Fatal(err)
1879+
}
1880+
1881+
if podCapacity != 150 {
1882+
t.Fatalf("expected pods capacity 150, got %d", podCapacity)
1883+
}
1884+
})
1885+
1886+
}

Diff for: ‎pkg/manifests/config.go

+65-16
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,28 @@ package manifests
1616

1717
import (
1818
"bytes"
19+
"context"
1920
"encoding/json"
2021
"fmt"
2122
"io"
23+
"math"
2224

2325
configv1 "github.com/openshift/api/config/v1"
2426
monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
27+
poperator "github.com/prometheus-operator/prometheus-operator/pkg/operator"
2528
v1 "k8s.io/api/core/v1"
2629
k8syaml "k8s.io/apimachinery/pkg/util/yaml"
2730
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
31+
"k8s.io/klog/v2"
2832
)
2933

3034
const (
3135
DefaultRetentionValue = "15d"
36+
37+
// Limit the body size from scrape queries
38+
// Assumptions: one node has in average 110 pods, each pod exposes 400 metrics, each metric is expressed by on average 250 bytes.
39+
// 1.5x the size for a safe margin, it rounds to 16MB (16,500,000 Bytes).
40+
minimalSizeLimit = 1.5 * 110 * 400 * 250
3241
)
3342

3443
type Config struct {
@@ -171,17 +180,18 @@ type RemoteWriteSpec struct {
171180
}
172181

173182
type PrometheusK8sConfig struct {
174-
LogLevel string `json:"logLevel"`
175-
Retention string `json:"retention"`
176-
NodeSelector map[string]string `json:"nodeSelector"`
177-
Tolerations []v1.Toleration `json:"tolerations"`
178-
Resources *v1.ResourceRequirements `json:"resources"`
179-
ExternalLabels map[string]string `json:"externalLabels"`
180-
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
181-
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
182-
TelemetryMatches []string `json:"-"`
183-
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
184-
QueryLogFile string `json:"queryLogFile"`
183+
LogLevel string `json:"logLevel"`
184+
Retention string `json:"retention"`
185+
NodeSelector map[string]string `json:"nodeSelector"`
186+
Tolerations []v1.Toleration `json:"tolerations"`
187+
Resources *v1.ResourceRequirements `json:"resources"`
188+
ExternalLabels map[string]string `json:"externalLabels"`
189+
VolumeClaimTemplate *monv1.EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate"`
190+
RemoteWrite []RemoteWriteSpec `json:"remoteWrite"`
191+
TelemetryMatches []string `json:"-"`
192+
AlertmanagerConfigs []AdditionalAlertmanagerConfig `json:"additionalAlertmanagerConfigs"`
193+
QueryLogFile string `json:"queryLogFile"`
194+
EnforcedBodySizeLimit string `json:"enforcedBodySizeLimit,omitempty"`
185195
}
186196

187197
type AdditionalAlertmanagerConfig struct {
@@ -313,19 +323,18 @@ func (cfg *TelemeterClientConfig) IsEnabled() bool {
313323
return true
314324
}
315325

316-
func NewConfig(content io.Reader) (*Config, error) {
326+
func NewConfig(content io.Reader) (res *Config, err error) {
317327
c := Config{}
318328
cmc := ClusterMonitoringConfiguration{}
319-
err := k8syaml.NewYAMLOrJSONDecoder(content, 4096).Decode(&cmc)
329+
err = k8syaml.NewYAMLOrJSONDecoder(content, 4096).Decode(&cmc)
320330
if err != nil {
321331
return nil, err
322332
}
323333
c.ClusterMonitoringConfiguration = &cmc
324-
res := &c
334+
res = &c
325335
res.applyDefaults()
326336
c.UserWorkloadConfiguration = NewDefaultUserWorkloadMonitoringConfig()
327-
328-
return res, nil
337+
return
329338
}
330339

331340
func (c *Config) applyDefaults() {
@@ -471,6 +480,46 @@ func (c *Config) NoProxy() string {
471480
return c.ClusterMonitoringConfiguration.HTTPConfig.NoProxy
472481
}
473482

483+
// PodCapacityReader returns the maximum number of pods that can be scheduled in a cluster.
484+
type PodCapacityReader interface {
485+
PodCapacity(context.Context) (int, error)
486+
}
487+
488+
func (c *Config) LoadEnforcedBodySizeLimit(pcr PodCapacityReader, ctx context.Context) error {
489+
if c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit == "automatic" {
490+
podCapacity, err := pcr.PodCapacity(ctx)
491+
if err != nil {
492+
return fmt.Errorf("error fetching pod capacity: %v", err)
493+
}
494+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = calculateBodySizeLimit(podCapacity)
495+
return nil
496+
}
497+
498+
if c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit != "" {
499+
return poperator.ValidateSizeField(c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit)
500+
}
501+
502+
return nil
503+
}
504+
505+
func (c *Config) UseMinimalEnforcedBodySizeLimit() {
506+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit = fmt.Sprintf("%dMB", int(math.Ceil(float64(minimalSizeLimit)/(1024*1024))))
507+
}
508+
509+
func calculateBodySizeLimit(podCapacity int) string {
510+
const samplesPerPod = 400 // 400 samples per pod
511+
const sizePerSample = 200 // 200 Bytes
512+
const loadFactorPercentage = 60 // assume 80% of the maximum pods capacity per node is used
513+
514+
bodySize := loadFactorPercentage * podCapacity / 100 * samplesPerPod * sizePerSample
515+
if bodySize < minimalSizeLimit {
516+
bodySize = minimalSizeLimit
517+
klog.Infof("Calculated scrape body size limit is too small, using default value %v instead", minimalSizeLimit)
518+
}
519+
520+
return fmt.Sprintf("%dMB", int(math.Ceil(float64(bodySize)/(1024*1024))))
521+
}
522+
474523
func NewConfigFromString(content string) (*Config, error) {
475524
if content == "" {
476525
return NewDefaultConfig(), nil

Diff for: ‎pkg/manifests/config_test.go

+100-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ package manifests
1616

1717
import (
1818
"bytes"
19+
"context"
20+
"errors"
1921
"io/ioutil"
2022
"os"
2123
"testing"
@@ -192,7 +194,7 @@ func TestHttpProxyConfig(t *testing.T) {
192194
conf := `http:
193195
httpProxy: http://test.com
194196
httpsProxy: https://test.com
195-
noProxy: https://example.com
197+
noProxy: https://example.com
196198
`
197199

198200
c, err := NewConfig(bytes.NewBufferString(conf))
@@ -234,3 +236,100 @@ func TestHttpProxyConfig(t *testing.T) {
234236
}
235237
}
236238
}
239+
240+
type fakePodCapacity struct {
241+
capacity int
242+
err error
243+
}
244+
245+
func (fpc *fakePodCapacity) PodCapacity(context.Context) (int, error) {
246+
return fpc.capacity, fpc.err
247+
}
248+
249+
func TestLoadEnforcedBodySizeLimit(t *testing.T) {
250+
251+
mc_10 := fakePodCapacity{capacity: 10, err: nil}
252+
mc_1000 := fakePodCapacity{capacity: 1000, err: nil}
253+
mc_err := fakePodCapacity{capacity: 1000, err: errors.New("error")}
254+
for _, tt := range []struct {
255+
name string
256+
config string
257+
expectBodySizeLimit string
258+
expectError bool
259+
pcr PodCapacityReader
260+
}{
261+
{
262+
name: "empty config",
263+
config: "",
264+
expectBodySizeLimit: "",
265+
expectError: false,
266+
pcr: &mc_10,
267+
},
268+
{
269+
name: "disable body size limit",
270+
config: `{"prometheusK8s": {"enforcedBodySizeLimit": "0"}}`,
271+
expectBodySizeLimit: "0",
272+
expectError: false,
273+
pcr: &mc_10,
274+
},
275+
{
276+
name: "normal size format",
277+
config: `{"prometheusK8s": {"enforcedBodySizeLimit": "10KB"}}`,
278+
expectBodySizeLimit: "10KB",
279+
expectError: false,
280+
pcr: &mc_10,
281+
},
282+
{
283+
name: "invalid size format",
284+
config: `{"prometheusK8s": {"enforcedBodySizeLimit": "10EUR"}}`,
285+
expectBodySizeLimit: "",
286+
expectError: true,
287+
pcr: &mc_10,
288+
},
289+
{
290+
name: "automatic deduced limit: error when getting pods capacity",
291+
config: `{"prometheusK8s": {"enforcedBodySizeLimit": "automatic"}}`,
292+
expectBodySizeLimit: "",
293+
expectError: true,
294+
pcr: &mc_err,
295+
},
296+
{
297+
name: "automatically deduced limit: minimal 16MB",
298+
config: `{"prometheusK8s": {"enforcedBodySizeLimit": "automatic"}}`,
299+
expectBodySizeLimit: "16MB",
300+
expectError: false,
301+
pcr: &mc_10,
302+
},
303+
{
304+
name: "automatically deduced limit: larger than minimal 16MB",
305+
config: `{"prometheusK8s": {"enforcedBodySizeLimit": "automatic"}}`,
306+
expectBodySizeLimit: "46MB",
307+
expectError: false,
308+
pcr: &mc_1000,
309+
},
310+
} {
311+
t.Run(tt.name, func(t *testing.T) {
312+
c, err := NewConfigFromString(tt.config)
313+
if err != nil {
314+
t.Fatalf("config parsing error")
315+
}
316+
317+
err = c.LoadEnforcedBodySizeLimit(tt.pcr, context.Background())
318+
if tt.expectError {
319+
if err == nil {
320+
t.Fatalf("expected error, got nil")
321+
}
322+
return
323+
}
324+
if err != nil {
325+
t.Fatalf("expected no error, got error %v", err)
326+
}
327+
328+
if c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit != tt.expectBodySizeLimit {
329+
t.Fatalf("incorrect EnforcedBodySizeLimit is set: got %s, expected %s",
330+
c.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit,
331+
tt.expectBodySizeLimit)
332+
}
333+
})
334+
}
335+
}

Diff for: ‎pkg/manifests/manifests.go

+4
Original file line numberDiff line numberDiff line change
@@ -1653,6 +1653,10 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.Config
16531653
p.Spec.Secrets = append(p.Spec.Secrets, getAdditionalAlertmanagerSecrets(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.AlertmanagerConfigs)...)
16541654
}
16551655

1656+
if f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit != "" {
1657+
p.Spec.EnforcedBodySizeLimit = f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.EnforcedBodySizeLimit
1658+
}
1659+
16561660
return p, nil
16571661
}
16581662

0 commit comments

Comments
 (0)