Skip to content

Commit 62555ca

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#130701 from roycaihw/psi-metrics
Surface Pressure Stall Information (PSI) metrics
2 parents 78e9ba9 + 914a4ba commit 62555ca

20 files changed

+2159
-580
lines changed

‎pkg/features/kube_features.go

+10
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,12 @@ const (
398398
// Enable POD resources API with Get method
399399
KubeletPodResourcesGet featuregate.Feature = "KubeletPodResourcesGet"
400400

401+
// KubeletPSI enables Kubelet to surface PSI metrics
402+
// owner: @roycaihw
403+
// kep: https://kep.k8s.io/4205
404+
// alpha: v1.33
405+
KubeletPSI featuregate.Feature = "KubeletPSI"
406+
401407
// owner: @kannon92
402408
// kep: https://kep.k8s.io/4191
403409
//
@@ -1421,6 +1427,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
14211427
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
14221428
},
14231429

1430+
KubeletPSI: {
1431+
{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha},
1432+
},
1433+
14241434
KubeletRegistrationGetOnExistsOnly: {
14251435
{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA},
14261436
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Deprecated},

‎pkg/kubelet/cadvisor/cadvisor_linux.go

+6
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ import (
3939
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
4040
"github.com/google/cadvisor/manager"
4141
"github.com/google/cadvisor/utils/sysfs"
42+
utilfeature "k8s.io/apiserver/pkg/util/feature"
4243
"k8s.io/klog/v2"
44+
"k8s.io/kubernetes/pkg/features"
4345
"k8s.io/utils/ptr"
4446
)
4547

@@ -93,6 +95,10 @@ func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots [
9395
cadvisormetrics.OOMMetrics: struct{}{},
9496
}
9597

98+
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
99+
includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
100+
}
101+
96102
if usingLegacyStats || localStorageCapacityIsolation {
97103
includedMetrics[cadvisormetrics.DiskUsageMetrics] = struct{}{}
98104
}

‎pkg/kubelet/server/server.go

+5
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,11 @@ func (s *Server) InstallAuthNotRequiredHandlers() {
455455
cadvisormetrics.ProcessMetrics: struct{}{},
456456
cadvisormetrics.OOMMetrics: struct{}{},
457457
}
458+
459+
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
460+
includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
461+
}
462+
458463
// cAdvisor metrics are exposed under the secured handler as well
459464
r := compbasemetrics.NewKubeRegistry()
460465
r.RawMustRegister(metrics.NewPrometheusMachineCollector(prometheusHostAdapter{s.host}, includedMetrics))

‎pkg/kubelet/server/stats/summary.go

+5
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ import (
2424
"k8s.io/klog/v2"
2525

2626
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
utilfeature "k8s.io/apiserver/pkg/util/feature"
2728
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
29+
"k8s.io/kubernetes/pkg/features"
2830
"k8s.io/kubernetes/pkg/kubelet/util"
2931
)
3032

@@ -113,6 +115,9 @@ func (sp *summaryProviderImpl) Get(ctx context.Context, updateStats bool) (*stat
113115
Rlimit: rlimit,
114116
SystemContainers: sp.GetSystemContainersStats(nodeConfig, podStats, updateStats),
115117
}
118+
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
119+
nodeStats.IO = rootStats.IO
120+
}
116121
summary := statsapi.Summary{
117122
Node: nodeStats,
118123
Pods: podStats,

‎pkg/kubelet/server/stats/summary_test.go

+17
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ import (
2929

3030
v1 "k8s.io/api/core/v1"
3131
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
32+
utilfeature "k8s.io/apiserver/pkg/util/feature"
33+
featuregatetesting "k8s.io/component-base/featuregate/testing"
3234
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
35+
"k8s.io/kubernetes/pkg/features"
3336
"k8s.io/kubernetes/pkg/kubelet/cm"
3437
statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing"
3538
)
@@ -48,6 +51,7 @@ var (
4851
)
4952

5053
func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
54+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
5155
ctx := context.Background()
5256
assert := assert.New(t)
5357

@@ -98,6 +102,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
98102
assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU)
99103
assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory)
100104
assert.Equal(summary.Node.Swap, cgroupStatsMap["/"].cs.Swap)
105+
assert.Equal(summary.Node.IO, cgroupStatsMap["/"].cs.IO)
101106
assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns)
102107
assert.Equal(summary.Node.Fs, rootFsStats)
103108
assert.Equal(&statsapi.RuntimeStats{ContainerFs: imageFsStats, ImageFs: imageFsStats}, summary.Node.Runtime)
@@ -111,6 +116,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
111116
Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators,
112117
UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics,
113118
Swap: cgroupStatsMap["/kubelet"].cs.Swap,
119+
IO: cgroupStatsMap["/kubelet"].cs.IO,
114120
})
115121
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
116122
Name: "misc",
@@ -120,6 +126,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
120126
Accelerators: cgroupStatsMap["/misc"].cs.Accelerators,
121127
UserDefinedMetrics: cgroupStatsMap["/misc"].cs.UserDefinedMetrics,
122128
Swap: cgroupStatsMap["/misc"].cs.Swap,
129+
IO: cgroupStatsMap["/misc"].cs.IO,
123130
})
124131
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
125132
Name: "runtime",
@@ -129,6 +136,7 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
129136
Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators,
130137
UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics,
131138
Swap: cgroupStatsMap["/runtime"].cs.Swap,
139+
IO: cgroupStatsMap["/runtime"].cs.IO,
132140
})
133141
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
134142
Name: "pods",
@@ -138,11 +146,13 @@ func TestSummaryProviderGetStatsNoSplitFileSystem(t *testing.T) {
138146
Accelerators: cgroupStatsMap["/pods"].cs.Accelerators,
139147
UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics,
140148
Swap: cgroupStatsMap["/pods"].cs.Swap,
149+
IO: cgroupStatsMap["/pods"].cs.IO,
141150
})
142151
assert.Equal(summary.Pods, podStats)
143152
}
144153

145154
func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
155+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
146156
ctx := context.Background()
147157
assert := assert.New(t)
148158

@@ -194,6 +204,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
194204
assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU)
195205
assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory)
196206
assert.Equal(summary.Node.Swap, cgroupStatsMap["/"].cs.Swap)
207+
assert.Equal(summary.Node.IO, cgroupStatsMap["/"].cs.IO)
197208
assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns)
198209
assert.Equal(summary.Node.Fs, rootFsStats)
199210
// Since we are a split filesystem we want root filesystem to be container fs and image to be image filesystem
@@ -208,6 +219,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
208219
Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators,
209220
UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics,
210221
Swap: cgroupStatsMap["/kubelet"].cs.Swap,
222+
IO: cgroupStatsMap["/kubelet"].cs.IO,
211223
})
212224
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
213225
Name: "misc",
@@ -217,6 +229,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
217229
Accelerators: cgroupStatsMap["/misc"].cs.Accelerators,
218230
UserDefinedMetrics: cgroupStatsMap["/misc"].cs.UserDefinedMetrics,
219231
Swap: cgroupStatsMap["/misc"].cs.Swap,
232+
IO: cgroupStatsMap["/misc"].cs.IO,
220233
})
221234
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
222235
Name: "runtime",
@@ -226,6 +239,7 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
226239
Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators,
227240
UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics,
228241
Swap: cgroupStatsMap["/runtime"].cs.Swap,
242+
IO: cgroupStatsMap["/runtime"].cs.IO,
229243
})
230244
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
231245
Name: "pods",
@@ -235,11 +249,13 @@ func TestSummaryProviderGetStatsSplitImageFs(t *testing.T) {
235249
Accelerators: cgroupStatsMap["/pods"].cs.Accelerators,
236250
UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics,
237251
Swap: cgroupStatsMap["/pods"].cs.Swap,
252+
IO: cgroupStatsMap["/pods"].cs.IO,
238253
})
239254
assert.Equal(summary.Pods, podStats)
240255
}
241256

242257
func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) {
258+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
243259
ctx := context.Background()
244260
assert := assert.New(t)
245261

@@ -283,6 +299,7 @@ func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) {
283299
assert.Nil(summary.Node.Network)
284300
assert.Nil(summary.Node.Fs)
285301
assert.Nil(summary.Node.Runtime)
302+
assert.Nil(summary.Node.IO)
286303

287304
assert.Len(summary.Node.SystemContainers, 4)
288305
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{

‎pkg/kubelet/stats/cadvisor_stats_provider.go

+3
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,9 @@ func (p *cadvisorStatsProvider) ListPodStats(ctx context.Context) ([]statsapi.Po
156156
podStats.CPU = cpu
157157
podStats.Memory = memory
158158
podStats.Swap = cadvisorInfoToSwapStats(podInfo)
159+
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
160+
podStats.IO = cadvisorInfoToIOStats(podInfo)
161+
}
159162
// ProcessStats were accumulated as the containers were iterated.
160163
}
161164

‎pkg/kubelet/stats/cadvisor_stats_provider_test.go

+15
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) {
114114
}
115115

116116
func TestCadvisorListPodStats(t *testing.T) {
117+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
117118
ctx := context.Background()
118119
const (
119120
namespace0 = "test0"
@@ -295,12 +296,14 @@ func TestCadvisorListPodStats(t *testing.T) {
295296
checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
296297
checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)
297298
checkSwapStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Swap)
299+
checkIOStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.IO)
298300

299301
con = indexCon[cName01]
300302
assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
301303
checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
302304
checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)
303305
checkSwapStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Swap)
306+
checkIOStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.IO)
304307

305308
assert.EqualValues(t, p0Time.Unix(), ps.StartTime.Time.Unix())
306309
checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
@@ -315,6 +318,9 @@ func TestCadvisorListPodStats(t *testing.T) {
315318
checkSwapStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Swap)
316319
checkContainersSwapStats(t, ps, infos["/pod0-c0"], infos["/pod0-c1"])
317320
}
321+
if ps.IO != nil {
322+
checkIOStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.IO)
323+
}
318324

319325
// Validate Pod1 Results
320326
ps, found = indexPods[prf1]
@@ -325,6 +331,7 @@ func TestCadvisorListPodStats(t *testing.T) {
325331
checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
326332
checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
327333
checkSwapStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Swap)
334+
checkIOStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.IO)
328335
checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network)
329336
checkContainersSwapStats(t, ps, infos["/pod1-c0"])
330337

@@ -337,6 +344,7 @@ func TestCadvisorListPodStats(t *testing.T) {
337344
checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
338345
checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
339346
checkSwapStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Swap)
347+
checkIOStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.IO)
340348
checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
341349
checkContainersSwapStats(t, ps, infos["/pod2-c0"])
342350

@@ -355,10 +363,12 @@ func TestCadvisorListPodStats(t *testing.T) {
355363
checkCPUStats(t, "Pod3Container1", seedPod3Container1, con.CPU)
356364
checkMemoryStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Memory)
357365
checkSwapStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Swap)
366+
checkIOStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.IO)
358367
checkContainersSwapStats(t, ps, infos["/pod3-c1"])
359368
}
360369

361370
func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
371+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletPSI, true)
362372
ctx := context.Background()
363373
const (
364374
namespace0 = "test0"
@@ -476,6 +486,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
476486
assert.Nil(t, con.Logs)
477487
assert.Nil(t, con.Accelerators)
478488
assert.Nil(t, con.UserDefinedMetrics)
489+
assert.Nil(t, con.IO)
479490

480491
con = indexCon[cName01]
481492
assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
@@ -486,11 +497,13 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
486497
assert.Nil(t, con.Logs)
487498
assert.Nil(t, con.Accelerators)
488499
assert.Nil(t, con.UserDefinedMetrics)
500+
assert.Nil(t, con.IO)
489501

490502
assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
491503
assert.Nil(t, ps.EphemeralStorage)
492504
assert.Nil(t, ps.VolumeStats)
493505
assert.Nil(t, ps.Network)
506+
assert.Nil(t, con.IO)
494507
if ps.CPU != nil {
495508
checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
496509
}
@@ -515,6 +528,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
515528
assert.Nil(t, ps.EphemeralStorage)
516529
assert.Nil(t, ps.VolumeStats)
517530
assert.Nil(t, ps.Network)
531+
assert.Nil(t, con.IO)
518532

519533
// Validate Pod2 Results
520534
ps, found = indexPods[prf2]
@@ -529,6 +543,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
529543
assert.Nil(t, ps.EphemeralStorage)
530544
assert.Nil(t, ps.VolumeStats)
531545
assert.Nil(t, ps.Network)
546+
assert.Nil(t, con.IO)
532547
}
533548

534549
func TestCadvisorImagesFsStatsKubeletSeparateDiskOff(t *testing.T) {

0 commit comments

Comments
 (0)