Skip to content

Commit d4d79f0

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#126213 from bitoku/kep2371-e2e
[KEP-2371] add test about container metrics from cadvisor
2 parents 609bf00 + 568255a commit d4d79f0

File tree

2 files changed

+182
-30
lines changed

2 files changed

+182
-30
lines changed

‎test/e2e_node/container_metrics_test.go

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2enode
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"time"
23+
24+
"github.com/onsi/ginkgo/v2"
25+
"github.com/onsi/gomega"
26+
"github.com/onsi/gomega/gstruct"
27+
"github.com/onsi/gomega/types"
28+
29+
"k8s.io/kubernetes/test/e2e/framework"
30+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
31+
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
32+
admissionapi "k8s.io/pod-security-admission/api"
33+
)
34+
35+
var _ = SIGDescribe("ContainerMetrics", "[LinuxOnly]", framework.WithNodeConformance(), func() {
36+
f := framework.NewDefaultFramework("container-metrics")
37+
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
38+
ginkgo.Context("when querying /metrics/cadvisor", func() {
39+
ginkgo.BeforeEach(func(ctx context.Context) {
40+
createMetricsPods(ctx, f)
41+
})
42+
ginkgo.AfterEach(func(ctx context.Context) {
43+
removeMetricsPods(ctx, f)
44+
})
45+
ginkgo.It("should report container metrics", func(ctx context.Context) {
46+
keys := gstruct.Keys{}
47+
ctrMatches := map[string]types.GomegaMatcher{
48+
"container_blkio_device_usage_total": boundedSample(0, 10000000),
49+
"container_cpu_load_average_10s": boundedSample(0, 100),
50+
"container_cpu_system_seconds_total": boundedSample(0, 100),
51+
"container_cpu_usage_seconds_total": boundedSample(0, 100),
52+
"container_cpu_user_seconds_total": boundedSample(0, 100),
53+
"container_file_descriptors": boundedSample(0, 100),
54+
"container_fs_reads_bytes_total": boundedSample(0, 10000000),
55+
"container_fs_reads_total": boundedSample(0, 100),
56+
"container_fs_usage_bytes": boundedSample(0, 1000000),
57+
"container_fs_writes_bytes_total": boundedSample(0, 1000000),
58+
"container_fs_writes_total": boundedSample(0, 100),
59+
"container_last_seen": boundedSample(time.Now().Add(-maxStatsAge).Unix(), time.Now().Add(2*time.Minute).Unix()),
60+
"container_memory_cache": boundedSample(1*e2evolume.Kb, 10*e2evolume.Mb),
61+
"container_memory_failcnt": preciseSample(0),
62+
"container_memory_failures_total": boundedSample(0, 1000000),
63+
"container_memory_mapped_file": boundedSample(0, 10000000),
64+
"container_memory_max_usage_bytes": boundedSample(0, 80*e2evolume.Mb),
65+
"container_memory_rss": boundedSample(10*e2evolume.Kb, 80*e2evolume.Mb),
66+
"container_memory_swap": preciseSample(0),
67+
"container_memory_usage_bytes": boundedSample(10*e2evolume.Kb, 80*e2evolume.Mb),
68+
"container_memory_working_set_bytes": boundedSample(10*e2evolume.Kb, 80*e2evolume.Mb),
69+
"container_oom_events_total": preciseSample(0),
70+
"container_processes": boundedSample(0, 10),
71+
"container_sockets": boundedSample(0, 10),
72+
"container_spec_cpu_period": preciseSample(100000),
73+
"container_spec_cpu_shares": preciseSample(2),
74+
"container_spec_memory_limit_bytes": preciseSample(79998976),
75+
"container_spec_memory_reservation_limit_bytes": preciseSample(0),
76+
"container_spec_memory_swap_limit_bytes": preciseSample(0),
77+
"container_start_time_seconds": boundedSample(time.Now().Add(-maxStatsAge).Unix(), time.Now().Add(2*time.Minute).Unix()),
78+
"container_tasks_state": preciseSample(0),
79+
"container_threads": boundedSample(0, 10),
80+
"container_threads_max": boundedSample(0, 100000),
81+
"container_ulimits_soft": boundedSample(0, 10000000),
82+
}
83+
appendMatchesForContainer(f.Namespace.Name, pod0, pod1, "busybox-container", ctrMatches, keys, gstruct.AllowDuplicates|gstruct.IgnoreExtras)
84+
85+
ctrOptionalMatches := map[string]types.GomegaMatcher{
86+
"container_fs_io_current": boundedSample(0, 100),
87+
"container_fs_io_time_seconds_total": boundedSample(0, 100),
88+
"container_fs_io_time_weighted_seconds_total": boundedSample(0, 100),
89+
"container_fs_inodes_free": boundedSample(0, 10*e2evolume.Kb),
90+
"container_fs_inodes_total": boundedSample(0, 100),
91+
"container_fs_limit_bytes": boundedSample(100*e2evolume.Mb, 10*e2evolume.Tb),
92+
"container_fs_usage_bytes": boundedSample(0, 1000000),
93+
"container_fs_read_seconds_total": preciseSample(0),
94+
"container_fs_reads_merged_total": preciseSample(0),
95+
"container_fs_sector_reads_total": preciseSample(0),
96+
"container_fs_sector_writes_total": preciseSample(0),
97+
"container_fs_write_seconds_total": preciseSample(0),
98+
"container_fs_writes_merged_total": preciseSample(0),
99+
}
100+
// Missing from containerd, so set gstruct.IgnoreMissing
101+
// See https://github.com/google/cadvisor/issues/2785
102+
appendMatchesForContainer(f.Namespace.Name, pod0, pod1, "busybox-container", ctrOptionalMatches, keys, gstruct.AllowDuplicates|gstruct.IgnoreMissing|gstruct.IgnoreExtras)
103+
104+
podMatches := map[string]types.GomegaMatcher{
105+
"container_network_receive_bytes_total": boundedSample(10, 10*e2evolume.Mb),
106+
"container_network_receive_errors_total": boundedSample(0, 1000),
107+
"container_network_receive_packets_dropped_total": boundedSample(0, 1000),
108+
"container_network_receive_packets_total": boundedSample(0, 1000),
109+
"container_network_transmit_bytes_total": boundedSample(10, 10*e2evolume.Mb),
110+
"container_network_transmit_errors_total": boundedSample(0, 1000),
111+
"container_network_transmit_packets_dropped_total": boundedSample(0, 1000),
112+
"container_network_transmit_packets_total": boundedSample(0, 1000),
113+
}
114+
// TODO: determine why these are missing from containerd but not CRI-O
115+
appendMatchesForContainer(f.Namespace.Name, pod0, pod1, "POD", podMatches, keys, gstruct.AllowDuplicates|gstruct.IgnoreMissing|gstruct.IgnoreExtras)
116+
117+
matchResourceMetrics := gstruct.MatchKeys(gstruct.IgnoreExtras, keys)
118+
ginkgo.By("Giving pods a minute to start up and produce metrics")
119+
gomega.Eventually(ctx, getContainerMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
120+
ginkgo.By("Ensuring the metrics match the expectations a few more times")
121+
gomega.Consistently(ctx, getContainerMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
122+
})
123+
})
124+
})
125+
126+
func getContainerMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) {
127+
ginkgo.By("getting container metrics from cadvisor")
128+
return e2emetrics.GrabKubeletMetricsWithoutProxy(ctx, framework.TestContext.NodeName+":10255", "/metrics/cadvisor")
129+
}
130+
131+
func preciseSample(value interface{}) types.GomegaMatcher {
132+
return gstruct.PointTo(gstruct.MatchAllFields(gstruct.Fields{
133+
"Metric": gstruct.Ignore(),
134+
"Value": gomega.BeEquivalentTo(value),
135+
"Timestamp": gstruct.Ignore(),
136+
"Histogram": gstruct.Ignore(),
137+
}))
138+
}
139+
140+
func appendMatchesForContainer(ns, pod1, pod2, ctr string, matches map[string]types.GomegaMatcher, keys gstruct.Keys, options gstruct.Options) {
141+
for k, v := range matches {
142+
keys[k] = gstruct.MatchElements(containerID, options, gstruct.Elements{
143+
fmt.Sprintf("%s::%s::%s", ns, pod1, ctr): v,
144+
fmt.Sprintf("%s::%s::%s", ns, pod2, ctr): v,
145+
})
146+
}
147+
}

‎test/e2e_node/resource_metrics_test.go

+35-30
Original file line numberDiff line numberDiff line change
@@ -51,24 +51,7 @@ var _ = SIGDescribe("ResourceMetricsAPI", feature.ResourceMetrics, func() {
5151
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
5252
ginkgo.Context("when querying /resource/metrics", func() {
5353
ginkgo.BeforeEach(func(ctx context.Context) {
54-
ginkgo.By("Creating test pods to measure their resource usage")
55-
numRestarts := int32(1)
56-
pods := getSummaryTestPods(f, numRestarts, pod0, pod1)
57-
e2epod.NewPodClient(f).CreateBatch(ctx, pods)
58-
59-
ginkgo.By("restarting the containers to ensure container metrics are still being gathered after a container is restarted")
60-
gomega.Eventually(ctx, func(ctx context.Context) error {
61-
for _, pod := range pods {
62-
err := verifyPodRestartCount(ctx, f, pod.Name, len(pod.Spec.Containers), numRestarts)
63-
if err != nil {
64-
return err
65-
}
66-
}
67-
return nil
68-
}, time.Minute, 5*time.Second).Should(gomega.Succeed())
69-
70-
ginkgo.By("Waiting 15 seconds for cAdvisor to collect 2 stats points")
71-
time.Sleep(15 * time.Second)
54+
createMetricsPods(ctx, f)
7255
})
7356
ginkgo.It("should report resource usage through the resource metrics api", func(ctx context.Context) {
7457
ginkgo.By("Fetching node so we can match against an appropriate memory limit")
@@ -145,22 +128,29 @@ var _ = SIGDescribe("ResourceMetricsAPI", feature.ResourceMetrics, func() {
145128
gomega.Consistently(ctx, getResourceMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
146129
})
147130
ginkgo.AfterEach(func(ctx context.Context) {
148-
ginkgo.By("Deleting test pods")
149-
var zero int64 = 0
150-
e2epod.NewPodClient(f).DeleteSync(ctx, pod0, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
151-
e2epod.NewPodClient(f).DeleteSync(ctx, pod1, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
152-
if !ginkgo.CurrentSpecReport().Failed() {
153-
return
154-
}
155-
if framework.TestContext.DumpLogsOnFailure {
156-
e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf)
157-
}
158-
ginkgo.By("Recording processes in system cgroups")
159-
recordSystemCgroupProcesses(ctx)
131+
removeMetricsPods(ctx, f)
160132
})
161133
})
162134
})
163135

136+
func createMetricsPods(ctx context.Context, f *framework.Framework) {
137+
ginkgo.By("Creating test pods to measure their resource usage")
138+
numRestarts := int32(1)
139+
pods := getSummaryTestPods(f, numRestarts, pod0, pod1)
140+
e2epod.NewPodClient(f).CreateBatch(ctx, pods)
141+
142+
ginkgo.By("Restarting the containers to ensure container metrics are still being gathered after a container is restarted")
143+
gomega.Eventually(ctx, func(ctx context.Context) error {
144+
for _, pod := range pods {
145+
err := verifyPodRestartCount(ctx, f, pod.Name, len(pod.Spec.Containers), numRestarts)
146+
if err != nil {
147+
return err
148+
}
149+
}
150+
return nil
151+
}, time.Minute, 5*time.Second).Should(gomega.Succeed())
152+
}
153+
164154
func getResourceMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) {
165155
ginkgo.By("getting stable resource metrics API")
166156
return e2emetrics.GrabKubeletMetricsWithoutProxy(ctx, nodeNameOrIP()+":10255", "/metrics/resource")
@@ -230,3 +220,18 @@ func haveKeys(keys ...string) types.GomegaMatcher {
230220

231221
return matcher
232222
}
223+
224+
func removeMetricsPods(ctx context.Context, f *framework.Framework) {
225+
ginkgo.By("Deleting test pods")
226+
var zero int64 = 0
227+
e2epod.NewPodClient(f).DeleteSync(ctx, pod0, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
228+
e2epod.NewPodClient(f).DeleteSync(ctx, pod1, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
229+
if !ginkgo.CurrentSpecReport().Failed() {
230+
return
231+
}
232+
if framework.TestContext.DumpLogsOnFailure {
233+
e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf)
234+
}
235+
ginkgo.By("Recording processes in system cgroups")
236+
recordSystemCgroupProcesses(ctx)
237+
}

0 commit comments

Comments
 (0)