Skip to content

Commit 6b5afa3

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #6608 from ykakarap/runtimeruntime-sdk_lifecycle-hooks_before-hooks
✨ RuntimeSDK: BeforeClusterCreate, BeforeClusterUpgrade implementation
2 parents 9bb071e + abb8f5f commit 6b5afa3

15 files changed

+768
-26
lines changed

‎api/v1beta1/condition_consts.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,4 +277,8 @@ const (
277277
// TopologyReconciledMachineDeploymentsUpgradePendingReason (Severity=Info) documents reconciliation of a Cluster topology
278278
// not yet completed because at least one of the MachineDeployments is not yet updated to match the desired topology spec.
279279
TopologyReconciledMachineDeploymentsUpgradePendingReason = "MachineDeploymentsUpgradePending"
280+
281+
// TopologyReconciledHookBlockingReason (Severity=Info) documents reconciliation of a Cluster topology
282+
// not yet completed because at least one of the lifecycle hooks is blocking.
283+
TopologyReconciledHookBlockingReason = "LifecycleHookBlocking"
280284
)

‎controllers/alias.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
clustertopologycontroller "sigs.k8s.io/cluster-api/internal/controllers/topology/cluster"
3333
machinedeploymenttopologycontroller "sigs.k8s.io/cluster-api/internal/controllers/topology/machinedeployment"
3434
machinesettopologycontroller "sigs.k8s.io/cluster-api/internal/controllers/topology/machineset"
35+
runtimeclient "sigs.k8s.io/cluster-api/internal/runtime/client"
3536
)
3637

3738
// Following types provides access to reconcilers implemented in internal/controllers, thus
@@ -133,6 +134,8 @@ type ClusterTopologyReconciler struct {
133134
// race conditions caused by an outdated cache.
134135
APIReader client.Reader
135136

137+
RuntimeClient runtimeclient.Client
138+
136139
// WatchFilterValue is the label value used to filter events prior to reconciliation.
137140
WatchFilterValue string
138141

@@ -145,6 +148,7 @@ func (r *ClusterTopologyReconciler) SetupWithManager(ctx context.Context, mgr ct
145148
return (&clustertopologycontroller.Reconciler{
146149
Client: r.Client,
147150
APIReader: r.APIReader,
151+
RuntimeClient: r.RuntimeClient,
148152
UnstructuredCachingClient: r.UnstructuredCachingClient,
149153
WatchFilterValue: r.WatchFilterValue,
150154
}).SetupWithManager(ctx, mgr, options)

‎internal/controllers/topology/cluster/cluster_controller.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package cluster
1919
import (
2020
"context"
2121
"fmt"
22+
"time"
2223

2324
"github.com/pkg/errors"
2425
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -30,14 +31,19 @@ import (
3031
"sigs.k8s.io/controller-runtime/pkg/client"
3132
"sigs.k8s.io/controller-runtime/pkg/controller"
3233
"sigs.k8s.io/controller-runtime/pkg/handler"
34+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3335
"sigs.k8s.io/controller-runtime/pkg/source"
3436

3537
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3638
"sigs.k8s.io/cluster-api/api/v1beta1/index"
3739
"sigs.k8s.io/cluster-api/controllers/external"
40+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
41+
"sigs.k8s.io/cluster-api/feature"
3842
"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/patches"
3943
"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/scope"
4044
"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/structuredmerge"
45+
runtimecatalog "sigs.k8s.io/cluster-api/internal/runtime/catalog"
46+
runtimeclient "sigs.k8s.io/cluster-api/internal/runtime/client"
4147
"sigs.k8s.io/cluster-api/util"
4248
"sigs.k8s.io/cluster-api/util/annotations"
4349
"sigs.k8s.io/cluster-api/util/patch"
@@ -59,6 +65,8 @@ type Reconciler struct {
5965
// race conditions caused by an outdated cache.
6066
APIReader client.Reader
6167

68+
RuntimeClient runtimeclient.Client
69+
6270
// WatchFilterValue is the label value used to filter events prior to reconciliation.
6371
WatchFilterValue string
6472

@@ -207,6 +215,17 @@ func (r *Reconciler) reconcile(ctx context.Context, s *scope.Scope) (ctrl.Result
207215
return ctrl.Result{}, errors.Wrap(err, "error reading current state of the Cluster topology")
208216
}
209217

218+
// The cluster topology is yet to be created. Call the BeforeClusterCreate hook before proceeding.
219+
if feature.Gates.Enabled(feature.RuntimeSDK) {
220+
res, err := r.callBeforeClusterCreateHook(ctx, s)
221+
if err != nil {
222+
return reconcile.Result{}, err
223+
}
224+
if !res.IsZero() {
225+
return res, nil
226+
}
227+
}
228+
210229
// Setup watches for InfrastructureCluster and ControlPlane CRs when they exist.
211230
if err := r.setupDynamicWatches(ctx, s); err != nil {
212231
return ctrl.Result{}, errors.Wrap(err, "error creating dynamic watch")
@@ -223,6 +242,12 @@ func (r *Reconciler) reconcile(ctx context.Context, s *scope.Scope) (ctrl.Result
223242
return ctrl.Result{}, errors.Wrap(err, "error reconciling the Cluster topology")
224243
}
225244

245+
// requeueAfter will not be 0 if any of the runtime hooks returns a blocking response.
246+
requeueAfter := s.HookResponseTracker.AggregateRetryAfter()
247+
if requeueAfter != 0 {
248+
return ctrl.Result{RequeueAfter: requeueAfter}, nil
249+
}
250+
226251
return ctrl.Result{}, nil
227252
}
228253

@@ -247,6 +272,25 @@ func (r *Reconciler) setupDynamicWatches(ctx context.Context, s *scope.Scope) er
247272
return nil
248273
}
249274

275+
func (r *Reconciler) callBeforeClusterCreateHook(ctx context.Context, s *scope.Scope) (reconcile.Result, error) {
276+
// If the cluster objects (InfraCluster, ControlPlane, etc) are not yet created we are in the creation phase.
277+
// Call the BeforeClusterCreate hook before proceeding.
278+
if s.Current.Cluster.Spec.InfrastructureRef == nil && s.Current.Cluster.Spec.ControlPlaneRef == nil {
279+
hookRequest := &runtimehooksv1.BeforeClusterCreateRequest{
280+
Cluster: *s.Current.Cluster,
281+
}
282+
hookResponse := &runtimehooksv1.BeforeClusterCreateResponse{}
283+
if err := r.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.BeforeClusterCreate, s.Current.Cluster, hookRequest, hookResponse); err != nil {
284+
return ctrl.Result{}, errors.Wrapf(err, "error calling the %s hook", runtimecatalog.HookName(runtimehooksv1.BeforeClusterCreate))
285+
}
286+
s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterCreate, hookResponse)
287+
if hookResponse.RetryAfterSeconds != 0 {
288+
return ctrl.Result{RequeueAfter: time.Duration(hookResponse.RetryAfterSeconds) * time.Second}, nil
289+
}
290+
}
291+
return ctrl.Result{}, nil
292+
}
293+
250294
// clusterClassToCluster is a handler.ToRequestsFunc to be used to enqueue requests for reconciliation
251295
// for Cluster to update when its own ClusterClass gets updated.
252296
func (r *Reconciler) clusterClassToCluster(o client.Object) []ctrl.Request {

‎internal/controllers/topology/cluster/cluster_controller_test.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,17 @@ import (
2626
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2727
"k8s.io/apimachinery/pkg/runtime/schema"
2828
utilfeature "k8s.io/component-base/featuregate/testing"
29+
ctrl "sigs.k8s.io/controller-runtime"
2930
"sigs.k8s.io/controller-runtime/pkg/client"
31+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3032

3133
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
34+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
3235
"sigs.k8s.io/cluster-api/feature"
3336
"sigs.k8s.io/cluster-api/internal/contract"
37+
"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/scope"
38+
runtimecatalog "sigs.k8s.io/cluster-api/internal/runtime/catalog"
39+
fakeruntimeclient "sigs.k8s.io/cluster-api/internal/runtime/client/fake"
3440
"sigs.k8s.io/cluster-api/internal/test/builder"
3541
"sigs.k8s.io/cluster-api/util/conditions"
3642
"sigs.k8s.io/cluster-api/util/patch"
@@ -436,6 +442,95 @@ func TestClusterReconciler_deleteClusterClass(t *testing.T) {
436442
g.Expect(env.Delete(ctx, clusterClass)).NotTo(Succeed())
437443
}
438444

445+
func TestReconciler_callBeforeClusterCreateHook(t *testing.T) {
446+
catalog := runtimecatalog.New()
447+
_ = runtimehooksv1.AddToCatalog(catalog)
448+
gvh, err := catalog.GroupVersionHook(runtimehooksv1.BeforeClusterCreate)
449+
if err != nil {
450+
panic(err)
451+
}
452+
453+
blockingResponse := &runtimehooksv1.BeforeClusterCreateResponse{
454+
CommonRetryResponse: runtimehooksv1.CommonRetryResponse{
455+
CommonResponse: runtimehooksv1.CommonResponse{
456+
Status: runtimehooksv1.ResponseStatusSuccess,
457+
},
458+
RetryAfterSeconds: int32(10),
459+
},
460+
}
461+
nonBlockingResponse := &runtimehooksv1.BeforeClusterCreateResponse{
462+
CommonRetryResponse: runtimehooksv1.CommonRetryResponse{
463+
CommonResponse: runtimehooksv1.CommonResponse{
464+
Status: runtimehooksv1.ResponseStatusSuccess,
465+
},
466+
RetryAfterSeconds: int32(0),
467+
},
468+
}
469+
failingResponse := &runtimehooksv1.BeforeClusterCreateResponse{
470+
CommonRetryResponse: runtimehooksv1.CommonRetryResponse{
471+
CommonResponse: runtimehooksv1.CommonResponse{
472+
Status: runtimehooksv1.ResponseStatusFailure,
473+
},
474+
},
475+
}
476+
477+
tests := []struct {
478+
name string
479+
hookResponse *runtimehooksv1.BeforeClusterCreateResponse
480+
wantResult reconcile.Result
481+
wantErr bool
482+
}{
483+
{
484+
name: "should return a requeue response when the BeforeClusterCreate hook is blocking",
485+
hookResponse: blockingResponse,
486+
wantResult: ctrl.Result{RequeueAfter: time.Duration(10) * time.Second},
487+
wantErr: false,
488+
},
489+
{
490+
name: "should return an empty response when the BeforeClusterCreate hook is not blocking",
491+
hookResponse: nonBlockingResponse,
492+
wantResult: ctrl.Result{},
493+
wantErr: false,
494+
},
495+
{
496+
name: "should error when the BeforeClusterCreate hook returns a failure response",
497+
hookResponse: failingResponse,
498+
wantResult: ctrl.Result{},
499+
wantErr: true,
500+
},
501+
}
502+
503+
for _, tt := range tests {
504+
t.Run(tt.name, func(t *testing.T) {
505+
g := NewWithT(t)
506+
507+
runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder().
508+
WithCatalog(catalog).
509+
WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{
510+
gvh: tt.hookResponse,
511+
}).
512+
Build()
513+
514+
r := &Reconciler{
515+
RuntimeClient: runtimeClient,
516+
}
517+
s := &scope.Scope{
518+
Current: &scope.ClusterState{
519+
Cluster: &clusterv1.Cluster{},
520+
},
521+
HookResponseTracker: scope.NewHookResponseTracker(),
522+
}
523+
res, err := r.callBeforeClusterCreateHook(ctx, s)
524+
if tt.wantErr {
525+
g.Expect(err).NotTo(BeNil())
526+
} else {
527+
g.Expect(err).To(BeNil())
528+
g.Expect(res).To(Equal(tt.wantResult))
529+
}
530+
})
531+
}
532+
}
533+
439534
// setupTestEnvForIntegrationTests builds and then creates in the envtest API server all objects required at init time for each of the
440535
// integration tests in this file. This includes:
441536
// - a first clusterClass with all the related templates

‎internal/controllers/topology/cluster/conditions.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,21 @@ func (r *Reconciler) reconcileTopologyReconciledCondition(s *scope.Scope, cluste
5757
return nil
5858
}
5959

60+
// If any of the lifecycle hooks are blocking any part of the reconciliation then topology
61+
// is not considered as fully reconciled.
62+
if s.HookResponseTracker.AggregateRetryAfter() != 0 {
63+
conditions.Set(
64+
cluster,
65+
conditions.FalseCondition(
66+
clusterv1.TopologyReconciledCondition,
67+
clusterv1.TopologyReconciledHookBlockingReason,
68+
clusterv1.ConditionSeverityInfo,
69+
s.HookResponseTracker.AggregateMessage(),
70+
),
71+
)
72+
return nil
73+
}
74+
6075
// If either the Control Plane or any of the MachineDeployments are still pending to pick up the new version (generally
6176
// happens when upgrading the cluster) then the topology is not considered as fully reconciled.
6277
if s.UpgradeTracker.ControlPlane.PendingUpgrade || s.UpgradeTracker.MachineDeployments.PendingUpgrade() {

‎internal/controllers/topology/cluster/conditions_test.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
corev1 "k8s.io/api/core/v1"
2525

2626
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
27+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
2728
"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/scope"
2829
"sigs.k8s.io/cluster-api/internal/test/builder"
2930
"sigs.k8s.io/cluster-api/util/conditions"
@@ -47,6 +48,24 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
4748
wantConditionReason: clusterv1.TopologyReconcileFailedReason,
4849
wantErr: false,
4950
},
51+
{
52+
name: "should set the condition to false if the there is a blocking hook",
53+
reconcileErr: nil,
54+
cluster: &clusterv1.Cluster{},
55+
s: &scope.Scope{
56+
HookResponseTracker: func() *scope.HookResponseTracker {
57+
hrt := scope.NewHookResponseTracker()
58+
hrt.Add(runtimehooksv1.BeforeClusterUpgrade, &runtimehooksv1.BeforeClusterUpgradeResponse{
59+
CommonRetryResponse: runtimehooksv1.CommonRetryResponse{
60+
RetryAfterSeconds: int32(10),
61+
},
62+
})
63+
return hrt
64+
}(),
65+
},
66+
wantConditionStatus: corev1.ConditionFalse,
67+
wantConditionReason: clusterv1.TopologyReconciledHookBlockingReason,
68+
},
5069
{
5170
name: "should set the condition to false if new version is not picked up because control plane is provisioning",
5271
reconcileErr: nil,
@@ -71,6 +90,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
7190
ut.ControlPlane.IsProvisioning = true
7291
return ut
7392
}(),
93+
HookResponseTracker: scope.NewHookResponseTracker(),
7494
},
7595
wantConditionStatus: corev1.ConditionFalse,
7696
wantConditionReason: clusterv1.TopologyReconciledControlPlaneUpgradePendingReason,
@@ -100,6 +120,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
100120
ut.ControlPlane.IsUpgrading = true
101121
return ut
102122
}(),
123+
HookResponseTracker: scope.NewHookResponseTracker(),
103124
},
104125
wantConditionStatus: corev1.ConditionFalse,
105126
wantConditionReason: clusterv1.TopologyReconciledControlPlaneUpgradePendingReason,
@@ -129,6 +150,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
129150
ut.ControlPlane.IsScaling = true
130151
return ut
131152
}(),
153+
HookResponseTracker: scope.NewHookResponseTracker(),
132154
},
133155
wantConditionStatus: corev1.ConditionFalse,
134156
wantConditionReason: clusterv1.TopologyReconciledControlPlaneUpgradePendingReason,
@@ -170,6 +192,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
170192
ut.ControlPlane.PendingUpgrade = true
171193
return ut
172194
}(),
195+
HookResponseTracker: scope.NewHookResponseTracker(),
173196
},
174197
wantConditionStatus: corev1.ConditionFalse,
175198
wantConditionReason: clusterv1.TopologyReconciledControlPlaneUpgradePendingReason,
@@ -213,6 +236,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
213236
ut.MachineDeployments.MarkPendingUpgrade("md0-abc123")
214237
return ut
215238
}(),
239+
HookResponseTracker: scope.NewHookResponseTracker(),
216240
},
217241
wantConditionStatus: corev1.ConditionFalse,
218242
wantConditionReason: clusterv1.TopologyReconciledMachineDeploymentsUpgradePendingReason,
@@ -256,6 +280,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
256280
ut.MachineDeployments.MarkPendingUpgrade("md0-abc123")
257281
return ut
258282
}(),
283+
HookResponseTracker: scope.NewHookResponseTracker(),
259284
},
260285
wantConditionStatus: corev1.ConditionFalse,
261286
wantConditionReason: clusterv1.TopologyReconciledMachineDeploymentsUpgradePendingReason,
@@ -285,6 +310,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
285310
ut.ControlPlane.IsUpgrading = true
286311
return ut
287312
}(),
313+
HookResponseTracker: scope.NewHookResponseTracker(),
288314
},
289315
wantConditionStatus: corev1.ConditionTrue,
290316
},
@@ -313,6 +339,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
313339
ut.ControlPlane.IsScaling = true
314340
return ut
315341
}(),
342+
HookResponseTracker: scope.NewHookResponseTracker(),
316343
},
317344
wantConditionStatus: corev1.ConditionTrue,
318345
},
@@ -367,6 +394,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
367394
ut.MachineDeployments.MarkPendingUpgrade("md1-abc123")
368395
return ut
369396
}(),
397+
HookResponseTracker: scope.NewHookResponseTracker(),
370398
},
371399
wantConditionStatus: corev1.ConditionFalse,
372400
wantConditionReason: clusterv1.TopologyReconciledMachineDeploymentsUpgradePendingReason,
@@ -421,6 +449,7 @@ func TestReconcileTopologyReconciledCondition(t *testing.T) {
421449
ut.ControlPlane.PendingUpgrade = false
422450
return ut
423451
}(),
452+
HookResponseTracker: scope.NewHookResponseTracker(),
424453
},
425454
wantConditionStatus: corev1.ConditionTrue,
426455
},

0 commit comments

Comments
 (0)