Skip to content

Commit c1ee348

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #99254 from wzshiming/automated-cherry-pick-of-#98005-upstream-release-1.20
Automated cherry pick of #98005: Sync node status during kubelet node shutdown
2 parents 5fea98b + dff5593 commit c1ee348

File tree

5 files changed

+60
-10
lines changed

5 files changed

+60
-10
lines changed

‎pkg/kubelet/kubelet.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -817,7 +817,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
817817
v1.NamespaceNodeLease,
818818
util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName)))
819819

820-
klet.shutdownManager = nodeshutdown.NewManager(klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration)
820+
// setup node shutdown manager
821+
shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration)
822+
823+
klet.shutdownManager = shutdownManager
824+
klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler)
821825

822826
// Finally, put the most recent version of the config on the Kubelet, so
823827
// people can see how it was configured.

‎pkg/kubelet/nodeshutdown/BUILD

+15
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,12 @@ go_library(
1111
deps = select({
1212
"@io_bazel_rules_go//go/platform:aix": [
1313
"//pkg/kubelet/eviction:go_default_library",
14+
"//pkg/kubelet/lifecycle:go_default_library",
1415
],
1516
"@io_bazel_rules_go//go/platform:android": [
1617
"//pkg/features:go_default_library",
1718
"//pkg/kubelet/eviction:go_default_library",
19+
"//pkg/kubelet/lifecycle:go_default_library",
1820
"//pkg/kubelet/nodeshutdown/systemd:go_default_library",
1921
"//pkg/kubelet/types:go_default_library",
2022
"//pkg/kubelet/util/format:go_default_library",
@@ -26,25 +28,32 @@ go_library(
2628
],
2729
"@io_bazel_rules_go//go/platform:darwin": [
2830
"//pkg/kubelet/eviction:go_default_library",
31+
"//pkg/kubelet/lifecycle:go_default_library",
2932
],
3033
"@io_bazel_rules_go//go/platform:dragonfly": [
3134
"//pkg/kubelet/eviction:go_default_library",
35+
"//pkg/kubelet/lifecycle:go_default_library",
3236
],
3337
"@io_bazel_rules_go//go/platform:freebsd": [
3438
"//pkg/kubelet/eviction:go_default_library",
39+
"//pkg/kubelet/lifecycle:go_default_library",
3540
],
3641
"@io_bazel_rules_go//go/platform:illumos": [
3742
"//pkg/kubelet/eviction:go_default_library",
43+
"//pkg/kubelet/lifecycle:go_default_library",
3844
],
3945
"@io_bazel_rules_go//go/platform:ios": [
4046
"//pkg/kubelet/eviction:go_default_library",
47+
"//pkg/kubelet/lifecycle:go_default_library",
4148
],
4249
"@io_bazel_rules_go//go/platform:js": [
4350
"//pkg/kubelet/eviction:go_default_library",
51+
"//pkg/kubelet/lifecycle:go_default_library",
4452
],
4553
"@io_bazel_rules_go//go/platform:linux": [
4654
"//pkg/features:go_default_library",
4755
"//pkg/kubelet/eviction:go_default_library",
56+
"//pkg/kubelet/lifecycle:go_default_library",
4857
"//pkg/kubelet/nodeshutdown/systemd:go_default_library",
4958
"//pkg/kubelet/types:go_default_library",
5059
"//pkg/kubelet/util/format:go_default_library",
@@ -56,21 +65,27 @@ go_library(
5665
],
5766
"@io_bazel_rules_go//go/platform:nacl": [
5867
"//pkg/kubelet/eviction:go_default_library",
68+
"//pkg/kubelet/lifecycle:go_default_library",
5969
],
6070
"@io_bazel_rules_go//go/platform:netbsd": [
6171
"//pkg/kubelet/eviction:go_default_library",
72+
"//pkg/kubelet/lifecycle:go_default_library",
6273
],
6374
"@io_bazel_rules_go//go/platform:openbsd": [
6475
"//pkg/kubelet/eviction:go_default_library",
76+
"//pkg/kubelet/lifecycle:go_default_library",
6577
],
6678
"@io_bazel_rules_go//go/platform:plan9": [
6779
"//pkg/kubelet/eviction:go_default_library",
80+
"//pkg/kubelet/lifecycle:go_default_library",
6881
],
6982
"@io_bazel_rules_go//go/platform:solaris": [
7083
"//pkg/kubelet/eviction:go_default_library",
84+
"//pkg/kubelet/lifecycle:go_default_library",
7185
],
7286
"@io_bazel_rules_go//go/platform:windows": [
7387
"//pkg/kubelet/eviction:go_default_library",
88+
"//pkg/kubelet/lifecycle:go_default_library",
7489
],
7590
"//conditions:default": [],
7691
}),

‎pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go

+28-6
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,16 @@ import (
3131
"k8s.io/klog/v2"
3232
"k8s.io/kubernetes/pkg/features"
3333
"k8s.io/kubernetes/pkg/kubelet/eviction"
34+
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
3435
"k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd"
3536
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
3637
"k8s.io/kubernetes/pkg/kubelet/util/format"
3738
)
3839

3940
const (
40-
nodeShutdownReason = "Shutdown"
41-
nodeShutdownMessage = "Node is shutting, evicting pods"
41+
nodeShutdownReason = "Shutdown"
42+
nodeShutdownMessage = "Node is shutting, evicting pods"
43+
nodeShutdownNotAdmitMessage = "Node is in progress of shutting down, not admitting any new pods"
4244
)
4345

4446
var systemDbus = func() (dbusInhibiter, error) {
@@ -63,8 +65,9 @@ type Manager struct {
6365
shutdownGracePeriodRequested time.Duration
6466
shutdownGracePeriodCriticalPods time.Duration
6567

66-
getPods eviction.ActivePodsFunc
67-
killPod eviction.KillPodFunc
68+
getPods eviction.ActivePodsFunc
69+
killPod eviction.KillPodFunc
70+
syncNodeStatus func()
6871

6972
dbusCon dbusInhibiter
7073
inhibitLock systemd.InhibitLock
@@ -76,14 +79,30 @@ type Manager struct {
7679
}
7780

7881
// NewManager returns a new node shutdown manager.
79-
func NewManager(getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) *Manager {
80-
return &Manager{
82+
func NewManager(getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) {
83+
manager := &Manager{
8184
getPods: getPodsFunc,
8285
killPod: killPodFunc,
86+
syncNodeStatus: syncNodeStatus,
8387
shutdownGracePeriodRequested: shutdownGracePeriodRequested,
8488
shutdownGracePeriodCriticalPods: shutdownGracePeriodCriticalPods,
8589
clock: clock.RealClock{},
8690
}
91+
return manager, manager
92+
}
93+
94+
// Admit rejects all pods if node is shutting
95+
func (m *Manager) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
96+
nodeShuttingDown := m.ShutdownStatus() != nil
97+
98+
if nodeShuttingDown {
99+
return lifecycle.PodAdmitResult{
100+
Admit: false,
101+
Reason: nodeShutdownReason,
102+
Message: nodeShutdownNotAdmitMessage,
103+
}
104+
}
105+
return lifecycle.PodAdmitResult{Admit: true}
87106
}
88107

89108
// Start starts the node shutdown manager and will start watching the node for shutdown events.
@@ -158,6 +177,9 @@ func (m *Manager) Start() error {
158177
m.nodeShuttingDownMutex.Unlock()
159178

160179
if isShuttingDown {
180+
// Update node status and ready condition
181+
go m.syncNodeStatus()
182+
161183
m.processShutdownEvent()
162184
} else {
163185
m.aquireInhibitLock()

‎pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ func TestManager(t *testing.T) {
224224
}
225225
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, true)()
226226

227-
manager := NewManager(activePodsFunc, killPodsFunc, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods)
227+
manager, _ := NewManager(activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods)
228228
manager.clock = clock.NewFakeClock(time.Now())
229229

230230
err := manager.Start()
@@ -236,6 +236,7 @@ func TestManager(t *testing.T) {
236236
assert.NoError(t, err, "expected manager.Start() to not return error")
237237
assert.True(t, fakeDbus.didInhibitShutdown, "expected that manager inhibited shutdown")
238238
assert.NoError(t, manager.ShutdownStatus(), "expected that manager does not return error since shutdown is not active")
239+
assert.Equal(t, manager.Admit(nil).Admit, true)
239240

240241
// Send fake shutdown event
241242
fakeShutdownChan <- true
@@ -253,6 +254,7 @@ func TestManager(t *testing.T) {
253254
}
254255

255256
assert.Error(t, manager.ShutdownStatus(), "expected that manager returns error since shutdown is active")
257+
assert.Equal(t, manager.Admit(nil).Admit, false)
256258
assert.Equal(t, tc.expectedPodToGracePeriodOverride, killedPodsToGracePeriods)
257259
assert.Equal(t, tc.expectedDidOverrideInhibitDelay, fakeDbus.didOverrideInhibitDelay, "override system inhibit delay differs")
258260
}

‎pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,21 @@ import (
2222
"time"
2323

2424
"k8s.io/kubernetes/pkg/kubelet/eviction"
25+
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
2526
)
2627

2728
// Manager is a fake node shutdown manager for non linux platforms.
2829
type Manager struct{}
2930

3031
// NewManager returns a fake node shutdown manager for non linux platforms.
31-
func NewManager(getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) *Manager {
32-
return &Manager{}
32+
func NewManager(getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) {
33+
m := &Manager{}
34+
return m, m
35+
}
36+
37+
// Admit returns a fake Pod admission which always returns true
38+
func (m *Manager) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
39+
return lifecycle.PodAdmitResult{Admit: true}
3340
}
3441

3542
// Start is a no-op always returning nil for non linux platforms.

0 commit comments

Comments
 (0)