Skip to content

Commit 22ead4f

Browse files
killianmuldoonykakarap
and
ykakarap
committed
Add lifecycle hook handlers to test extension
Signed-off-by: killianmuldoon <[email protected]> Co-authored-by: ykakarap <[email protected]>
1 parent 8d7f010 commit 22ead4f

12 files changed

+366
-6
lines changed

test/e2e/cluster_upgrade_runtimesdk.go

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@ import (
2626

2727
. "github.com/onsi/ginkgo"
2828
. "github.com/onsi/gomega"
29+
"github.com/pkg/errors"
2930
corev1 "k8s.io/api/core/v1"
3031
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3132
"k8s.io/utils/pointer"
33+
"sigs.k8s.io/controller-runtime/pkg/client"
3234

3335
runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1"
3436
"sigs.k8s.io/cluster-api/test/framework"
@@ -117,22 +119,26 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
117119
})
118120

119121
It("Should create and upgrade a workload cluster", func() {
122+
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
120123
By("Deploy Test Extension")
121124
testExtensionDeploymentTemplate, err := os.ReadFile(testExtensionPath) //nolint:gosec
122-
Expect(err).ToNot(HaveOccurred(), "Failed to read the extension config deployment manifest file")
125+
Expect(err).ToNot(HaveOccurred(), "Failed to read the extension deployment manifest file")
123126

124127
// Set the SERVICE_NAMESPACE, which is used in the cert-manager Certificate CR.
125128
// We have to dynamically set the namespace here, because it depends on the test run and thus
126129
// cannot be set when rendering the test extension YAML with kustomize.
127130
testExtensionDeployment := strings.ReplaceAll(string(testExtensionDeploymentTemplate), "${SERVICE_NAMESPACE}", namespace.Name)
128-
Expect(testExtensionDeployment).ToNot(BeEmpty(), "Test Extension deployment manifest file should not be empty")
129131

132+
Expect(testExtensionDeployment).ToNot(BeEmpty(), "Test Extension deployment manifest file should not be empty")
130133
Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(testExtensionDeployment), "--namespace", namespace.Name)).To(Succeed())
131134

132-
By("Deploy Test Extension ExtensionConfig")
135+
By("Deploy Test Extension ExtensionConfig and ConfigMap")
133136
ext = extensionConfig(specName, namespace)
134137
err = input.BootstrapClusterProxy.GetClient().Create(ctx, ext)
135138
Expect(err).ToNot(HaveOccurred(), "Failed to create the extension config")
139+
responses := responsesConfigMap(clusterName, namespace)
140+
err = input.BootstrapClusterProxy.GetClient().Create(ctx, responses)
141+
Expect(err).ToNot(HaveOccurred(), "Failed to create the responses configmap")
136142

137143
By("Creating a workload cluster")
138144

@@ -145,7 +151,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
145151
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
146152
Flavor: pointer.StringDeref(input.Flavor, "upgrades"),
147153
Namespace: namespace.Name,
148-
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
154+
ClusterName: clusterName,
149155
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
150156
ControlPlaneMachineCount: pointer.Int64Ptr(controlPlaneMachineCount),
151157
WorkerMachineCount: pointer.Int64Ptr(workerMachineCount),
@@ -194,6 +200,17 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
194200
WaitForNodesReady: input.E2EConfig.GetIntervals(specName, "wait-nodes-ready"),
195201
})
196202

203+
By("Checking all lifecycle hooks have been called")
204+
// Assert that each hook passed to this function is marked as "true" in the response configmap
205+
err = checkLifecycleHooks(input.BootstrapClusterProxy.GetClient(), namespace.Name, clusterName, map[string]string{
206+
"BeforeClusterCreate-response": "",
207+
"BeforeClusterUpgrade-response": "",
208+
"AfterControlPlaneInitialized-response": "",
209+
"AfterControlPlaneUpgrade-response": "",
210+
"AfterClusterUpgrade-response": "",
211+
})
212+
Expect(err).ToNot(HaveOccurred(), "Lifecycle hook calls were not as expected")
213+
197214
By("PASSED!")
198215
})
199216

@@ -241,3 +258,40 @@ func extensionConfig(specName string, namespace *corev1.Namespace) *runtimev1.Ex
241258
},
242259
}
243260
}
261+
262+
// responsesConfigMap generates a ConfigMap with preloaded responses for the test extension.
263+
func responsesConfigMap(name string, namespace *corev1.Namespace) *corev1.ConfigMap {
264+
return &corev1.ConfigMap{
265+
ObjectMeta: metav1.ObjectMeta{
266+
Name: fmt.Sprintf("%s-hookresponses", name),
267+
Namespace: namespace.Name,
268+
},
269+
// Every responses contain only Status:Success. The test checks whether each handler has been called at least once.
270+
Data: map[string]string{
271+
"BeforeClusterCreate-response": "{\"Status\": \"Success\"}",
272+
"BeforeClusterUpgrade-response": "{\"Status\": \"Success\"}",
273+
"AfterControlPlaneInitialized-response": "{\"Status\": \"Success\"}",
274+
"AfterControlPlaneUpgrade-response": "{\"Status\": \"Success\"}",
275+
"AfterClusterUpgrade-response": "{\"Status\": \"Success\"}",
276+
},
277+
}
278+
}
279+
280+
func checkLifecycleHooks(c client.Client, namespace string, clusterName string, hooks map[string]string) error {
281+
configMap := &corev1.ConfigMap{}
282+
configMapName := clusterName + "-hookresponses"
283+
err := c.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap)
284+
Expect(err).ToNot(HaveOccurred(), "Failed to get the hook response configmap")
285+
for hook := range hooks {
286+
if _, ok := configMap.Data[hook]; !ok {
287+
return errors.Errorf("hook %s call not registered in configMap %s/%s", hook, namespace, configMapName)
288+
}
289+
for k, v := range configMap.Data {
290+
// If it's a response key but it does not have a success response throw an error.
291+
if strings.Contains(k, "-response") && v != "{\"Status\": \"Success\"}" {
292+
return errors.Errorf("hook %s call returned %v in configMap, expected: %s", k, v, "true")
293+
}
294+
}
295+
}
296+
return nil
297+
}

test/extension/config/default/extension.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ spec:
1919
image: controller:latest
2020
name: extension
2121
terminationGracePeriodSeconds: 10
22+
serviceAccountName: extension
2223
tolerations:
2324
- effect: NoSchedule
2425
key: node-role.kubernetes.io/master

test/extension/config/default/extension_image_patch.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ spec:
66
template:
77
spec:
88
containers:
9-
- image: gcr.io/k8s-staging-cluster-api/test-extension:main
9+
- image: gcr.io/k8s-staging-cluster-api/test-extension-arm64:dev
1010
name: extension
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: test-extension
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- image: gcr.io/k8s-staging-cluster-api/test-extension-arm64:dev
10+
name: extension

test/extension/config/default/extension_pull_policy.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ spec:
77
spec:
88
containers:
99
- name: extension
10-
imagePullPolicy: Always
10+
imagePullPolicy: IfNotPresent
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: test-extension
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: extension
10+
imagePullPolicy: IfNotPresent

test/extension/config/default/kustomization.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ commonLabels:
33
resources:
44
- extension.yaml
55
- service.yaml
6+
- role.yaml
7+
- rolebinding.yaml
8+
- service_account.yaml
69

710
bases:
811
- ../certmanager
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: extension
5+
rules:
6+
- apiGroups:
7+
- ""
8+
resources:
9+
- configmaps
10+
verbs:
11+
- get
12+
- list
13+
- watch
14+
- patch
15+
- update
16+
- create
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: extension
5+
roleRef:
6+
apiGroup: rbac.authorization.k8s.io
7+
kind: ClusterRole
8+
name: extension
9+
subjects:
10+
- kind: ServiceAccount
11+
name: extension
12+
namespace: ${SERVICE_NAMESPACE}
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: extension
Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package lifecycle contains the handlers for the lifecycle hooks.
18+
package lifecycle
19+
20+
import (
21+
"context"
22+
"fmt"
23+
"strconv"
24+
25+
"github.com/pkg/errors"
26+
"gopkg.in/yaml.v2"
27+
corev1 "k8s.io/api/core/v1"
28+
"k8s.io/apimachinery/pkg/types"
29+
ctrl "sigs.k8s.io/controller-runtime"
30+
"sigs.k8s.io/controller-runtime/pkg/client"
31+
32+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
33+
runtimecatalog "sigs.k8s.io/cluster-api/internal/runtime/catalog"
34+
)
35+
36+
// Handler is the handler for the lifecycle hooks.
37+
type Handler struct {
38+
Client client.Client
39+
}
40+
41+
// DoBeforeClusterCreate implements the BeforeClusterCreate hook.
42+
func (h *Handler) DoBeforeClusterCreate(ctx context.Context, request *runtimehooksv1.BeforeClusterCreateRequest, response *runtimehooksv1.BeforeClusterCreateResponse) {
43+
log := ctrl.LoggerFrom(ctx)
44+
log.Info("BeforeClusterCreate is called")
45+
cluster := request.Cluster
46+
if err := h.recordCallInConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterCreate); err != nil {
47+
response.Status = runtimehooksv1.ResponseStatusFailure
48+
response.Message = err.Error()
49+
return
50+
}
51+
log.Info("BeforeClusterCreate has been recorded in configmap", "cm", cluster.Name+"-hookresponses")
52+
53+
err := h.readResponseFromConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterCreate, response)
54+
if err != nil {
55+
response.Status = runtimehooksv1.ResponseStatusFailure
56+
response.Message = err.Error()
57+
return
58+
}
59+
60+
log.Info(fmt.Sprintf("BeforeClusterCreate responding RetryAfterSeconds: %v\n", response.RetryAfterSeconds))
61+
}
62+
63+
// DoBeforeClusterUpgrade implements the BeforeClusterUpgrade hook.
64+
func (h *Handler) DoBeforeClusterUpgrade(ctx context.Context, request *runtimehooksv1.BeforeClusterUpgradeRequest, response *runtimehooksv1.BeforeClusterUpgradeResponse) {
65+
log := ctrl.LoggerFrom(ctx)
66+
log.Info("BeforeClusterUpgrade is called")
67+
cluster := request.Cluster
68+
if err := h.recordCallInConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterUpgrade); err != nil {
69+
response.Status = runtimehooksv1.ResponseStatusFailure
70+
response.Message = err.Error()
71+
return
72+
}
73+
err := h.readResponseFromConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterUpgrade, response)
74+
if err != nil {
75+
response.Status = runtimehooksv1.ResponseStatusFailure
76+
response.Message = err.Error()
77+
return
78+
}
79+
log.Info(fmt.Sprintf("BeforeClusterUpgrade responding RetryAfterSeconds: %v\n", response.RetryAfterSeconds))
80+
}
81+
82+
// DoAfterControlPlaneInitialized implements the AfterControlPlaneInitialized hook.
83+
func (h *Handler) DoAfterControlPlaneInitialized(ctx context.Context, request *runtimehooksv1.AfterControlPlaneInitializedRequest, response *runtimehooksv1.AfterControlPlaneInitializedResponse) {
84+
log := ctrl.LoggerFrom(ctx)
85+
log.Info("AfterControlPlaneInitialized is called")
86+
cluster := request.Cluster
87+
if err := h.recordCallInConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneInitialized); err != nil {
88+
response.Status = runtimehooksv1.ResponseStatusFailure
89+
response.Message = err.Error()
90+
return
91+
}
92+
err := h.readResponseFromConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneInitialized, response)
93+
if err != nil {
94+
response.Status = runtimehooksv1.ResponseStatusFailure
95+
response.Message = err.Error()
96+
return
97+
}
98+
}
99+
100+
// DoAfterControlPlaneUpgrade implements the AfterControlPlaneUpgrade hook.
101+
func (h *Handler) DoAfterControlPlaneUpgrade(ctx context.Context, request *runtimehooksv1.AfterControlPlaneUpgradeRequest, response *runtimehooksv1.AfterControlPlaneUpgradeResponse) {
102+
log := ctrl.LoggerFrom(ctx)
103+
log.Info("AfterControlPlaneUpgrade is called")
104+
cluster := request.Cluster
105+
if err := h.recordCallInConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneUpgrade); err != nil {
106+
response.Status = runtimehooksv1.ResponseStatusFailure
107+
response.Message = err.Error()
108+
return
109+
}
110+
err := h.readResponseFromConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneUpgrade, response)
111+
if err != nil {
112+
response.Status = runtimehooksv1.ResponseStatusFailure
113+
response.Message = err.Error()
114+
return
115+
}
116+
log.Info(fmt.Sprintf("AfterControlPlaneUpgrade responding RetryAfterSeconds: %v\n", response.RetryAfterSeconds))
117+
}
118+
119+
// DoAfterClusterUpgrade implements the AfterClusterUpgrade hook.
120+
func (h *Handler) DoAfterClusterUpgrade(ctx context.Context, request *runtimehooksv1.AfterClusterUpgradeRequest, response *runtimehooksv1.AfterClusterUpgradeResponse) {
121+
log := ctrl.LoggerFrom(ctx)
122+
log.Info("AfterClusterUpgrade is called")
123+
cluster := request.Cluster
124+
if err := h.recordCallInConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.AfterClusterUpgrade); err != nil {
125+
response.Status = runtimehooksv1.ResponseStatusFailure
126+
response.Message = err.Error()
127+
return
128+
}
129+
err := h.readResponseFromConfigMap(cluster.Name, cluster.Namespace, runtimehooksv1.AfterClusterUpgrade, response)
130+
if err != nil {
131+
response.Status = runtimehooksv1.ResponseStatusFailure
132+
response.Message = err.Error()
133+
return
134+
}
135+
}
136+
137+
func (h *Handler) readResponseFromConfigMap(name, namespace string, hook runtimecatalog.Hook, response runtimehooksv1.ResponseObject) error {
138+
hookName := runtimecatalog.HookName(hook)
139+
configMap := &corev1.ConfigMap{}
140+
configMapName := name + "-hookresponses"
141+
if err := h.Client.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap); err != nil {
142+
return errors.Wrapf(err, "failed to read the ConfigMap %s/%s", namespace, configMapName)
143+
}
144+
m := map[string]string{}
145+
if err := yaml.Unmarshal([]byte(configMap.Data[hookName+"-response"]), m); err != nil {
146+
return errors.Wrapf(err, "failed to read %q response information from ConfigMap", hook)
147+
}
148+
// If the response was a success return early.
149+
if status, ok := m["Status"]; ok && status == string(runtimehooksv1.ResponseStatusSuccess) {
150+
response.SetStatus(runtimehooksv1.ResponseStatusSuccess)
151+
return nil
152+
}
153+
if _, ok := m["Message"]; ok {
154+
response.SetMessage(m["Message"])
155+
}
156+
if retryResponse, ok := response.(runtimehooksv1.RetryResponseObject); ok {
157+
retryAfterSeconds, err := strconv.Atoi(m["RetryAfterSeconds"]) //nolint:gosec
158+
if err != nil {
159+
return err
160+
}
161+
retryResponse.SetRetryAfterSeconds(int32(retryAfterSeconds))
162+
}
163+
return nil
164+
}
165+
166+
func (h *Handler) recordCallInConfigMap(name, namespace string, hook runtimecatalog.Hook) error {
167+
hookName := runtimecatalog.HookName(hook)
168+
configMap := &corev1.ConfigMap{}
169+
configMapName := name + "-hookresponses"
170+
if err := h.Client.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap); err != nil {
171+
return errors.Wrapf(err, "failed to read the ConfigMap %s/%s", namespace, configMapName)
172+
}
173+
174+
patch := client.RawPatch(types.MergePatchType,
175+
[]byte(fmt.Sprintf("{\"data\":{\"%s-called\":\"true\"}}", hookName)))
176+
if err := h.Client.Patch(context.Background(), configMap, patch); err != nil {
177+
return errors.Wrapf(err, "failed to update the ConfigMap %s/%s", namespace, configMapName)
178+
}
179+
return nil
180+
}

0 commit comments

Comments
 (0)