Skip to content

Commit 92644ba

Browse files
dmvoloddmvolod
committed
✨ Handle Kubernetes events for waiting CoreProvider in preflight check
# Conflicts: # go.mod
1 parent bfcacce commit 92644ba

17 files changed

+307
-91
lines changed

‎.golangci.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ linters:
2929
- errname
3030
- errorlint
3131
- exhaustive
32-
- exportloopref
32+
- copyloopvar
3333
- forcetypeassert
3434
- ginkgolinter
3535
- goconst

‎cmd/main.go

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
"context"
2021
"flag"
2122
"fmt"
2223
"os"
@@ -37,7 +38,7 @@ import (
3738
"sigs.k8s.io/cluster-api/util/flags"
3839
"sigs.k8s.io/cluster-api/version"
3940
ctrl "sigs.k8s.io/controller-runtime"
40-
cache "sigs.k8s.io/controller-runtime/pkg/cache"
41+
"sigs.k8s.io/controller-runtime/pkg/cache"
4142
"sigs.k8s.io/controller-runtime/pkg/client"
4243
"sigs.k8s.io/controller-runtime/pkg/controller"
4344
"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -194,7 +195,7 @@ func main() {
194195
ctx := ctrl.SetupSignalHandler()
195196

196197
setupChecks(mgr)
197-
setupReconcilers(mgr, watchConfigSecretChanges)
198+
setupReconcilers(ctx, mgr, watchConfigSecretChanges)
198199
setupWebhooks(mgr)
199200

200201
// +kubebuilder:scaffold:builder
@@ -218,14 +219,14 @@ func setupChecks(mgr ctrl.Manager) {
218219
}
219220
}
220221

221-
func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
222+
func setupReconcilers(ctx context.Context, mgr ctrl.Manager, watchConfigSecretChanges bool) {
222223
if err := (&providercontroller.GenericProviderReconciler{
223224
Provider: &operatorv1.CoreProvider{},
224225
ProviderList: &operatorv1.CoreProviderList{},
225226
Client: mgr.GetClient(),
226227
Config: mgr.GetConfig(),
227228
WatchConfigSecretChanges: watchConfigSecretChanges,
228-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
229+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
229230
setupLog.Error(err, "unable to create controller", "controller", "CoreProvider")
230231
os.Exit(1)
231232
}
@@ -236,7 +237,7 @@ func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
236237
Client: mgr.GetClient(),
237238
Config: mgr.GetConfig(),
238239
WatchConfigSecretChanges: watchConfigSecretChanges,
239-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
240+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
240241
setupLog.Error(err, "unable to create controller", "controller", "InfrastructureProvider")
241242
os.Exit(1)
242243
}
@@ -247,7 +248,7 @@ func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
247248
Client: mgr.GetClient(),
248249
Config: mgr.GetConfig(),
249250
WatchConfigSecretChanges: watchConfigSecretChanges,
250-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
251+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
251252
setupLog.Error(err, "unable to create controller", "controller", "BootstrapProvider")
252253
os.Exit(1)
253254
}
@@ -258,7 +259,7 @@ func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
258259
Client: mgr.GetClient(),
259260
Config: mgr.GetConfig(),
260261
WatchConfigSecretChanges: watchConfigSecretChanges,
261-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
262+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
262263
setupLog.Error(err, "unable to create controller", "controller", "ControlPlaneProvider")
263264
os.Exit(1)
264265
}
@@ -269,7 +270,7 @@ func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
269270
Client: mgr.GetClient(),
270271
Config: mgr.GetConfig(),
271272
WatchConfigSecretChanges: watchConfigSecretChanges,
272-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
273+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
273274
setupLog.Error(err, "unable to create controller", "controller", "AddonProvider")
274275
os.Exit(1)
275276
}
@@ -280,7 +281,7 @@ func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
280281
Client: mgr.GetClient(),
281282
Config: mgr.GetConfig(),
282283
WatchConfigSecretChanges: watchConfigSecretChanges,
283-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
284+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
284285
setupLog.Error(err, "unable to create controller", "controller", "IPAMProvider")
285286
os.Exit(1)
286287
}
@@ -291,7 +292,7 @@ func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
291292
Client: mgr.GetClient(),
292293
Config: mgr.GetConfig(),
293294
WatchConfigSecretChanges: watchConfigSecretChanges,
294-
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
295+
}).SetupWithManager(ctx, mgr, concurrency(concurrencyNumber)); err != nil {
295296
setupLog.Error(err, "unable to create controller", "controller", "RuntimeExtensionProvider")
296297
os.Exit(1)
297298
}

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.23
44

55
require (
66
github.com/MakeNowJust/heredoc v1.0.0
7+
github.com/Masterminds/goutils v1.1.1
78
github.com/evanphx/json-patch/v5 v5.9.11
89
github.com/go-errors/errors v1.5.1
910
github.com/go-logr/logr v1.4.2
@@ -30,7 +31,6 @@ require (
3031

3132
require (
3233
dario.cat/mergo v1.0.1 // indirect
33-
github.com/Masterminds/goutils v1.1.1 // indirect
3434
github.com/Masterminds/semver/v3 v3.3.0 // indirect
3535
github.com/Masterminds/sprig/v3 v3.3.0 // indirect
3636
github.com/NYTimes/gziphandler v1.1.1 // indirect

‎internal/controller/consts.go

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,7 @@ limitations under the License.
1616

1717
package controller
1818

19-
import "time"
20-
2119
const (
22-
// preflightFailedRequeueAfter is how long to wait before trying to reconcile
23-
// if some preflight check has failed.
24-
preflightFailedRequeueAfter = 30 * time.Second
25-
2620
// configPath is the path to the clusterctl config file.
2721
configPath = "/config/clusterctl.yaml"
2822
)

‎internal/controller/coreprovider_to_providers.go

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
24+
"sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider"
25+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
26+
"sigs.k8s.io/cluster-api/util/conditions"
27+
ctrl "sigs.k8s.io/controller-runtime"
28+
"sigs.k8s.io/controller-runtime/pkg/client"
29+
"sigs.k8s.io/controller-runtime/pkg/handler"
30+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
31+
)
32+
33+
// newCoreProviderToProviderFuncMapForProviderList maps a ready CoreProvider object to all other provider objects.
34+
// It lists all the providers and if its PreflightCheckCondition is not True, this object will be added to the resulting request.
35+
// This means that notifications will only be sent to those objects that have not pass PreflightCheck.
36+
func newCoreProviderToProviderFuncMapForProviderList(k8sClient client.Client, providerList genericprovider.GenericProviderList) handler.MapFunc {
37+
providerListType := fmt.Sprintf("%t", providerList)
38+
39+
return func(ctx context.Context, obj client.Object) []reconcile.Request {
40+
log := ctrl.LoggerFrom(ctx).WithValues("provider", map[string]string{"name": obj.GetName(), "namespace": obj.GetNamespace()}, "providerListType", providerListType)
41+
coreProvider, ok := obj.(*operatorv1.CoreProvider)
42+
43+
if !ok {
44+
log.Error(fmt.Errorf("expected a %T but got a %T", operatorv1.CoreProvider{}, obj), "unable to cast object")
45+
return nil
46+
}
47+
48+
// We don't want to raise events if CoreProvider is not ready yet.
49+
if !conditions.IsTrue(coreProvider, clusterv1.ReadyCondition) {
50+
return nil
51+
}
52+
53+
var requests []reconcile.Request
54+
55+
if err := k8sClient.List(ctx, providerList); err != nil {
56+
log.Error(err, "failed to list providers")
57+
return nil
58+
}
59+
60+
for _, provider := range providerList.GetItems() {
61+
if !conditions.IsTrue(provider, operatorv1.PreflightCheckCondition) {
62+
// Raise secondary events for the providers that fail PreflightCheck.
63+
requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(provider)})
64+
}
65+
}
66+
67+
return requests
68+
}
69+
}

‎internal/controller/coreprovider_to_providers_test.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"testing"
21+
22+
. "github.com/onsi/gomega"
23+
corev1 "k8s.io/api/core/v1"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25+
"k8s.io/apimachinery/pkg/types"
26+
operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
27+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
28+
ctrl "sigs.k8s.io/controller-runtime"
29+
"sigs.k8s.io/controller-runtime/pkg/client"
30+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
31+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
32+
)
33+
34+
func TestCoreProviderToProvidersMapper(t *testing.T) {
35+
g := NewWithT(t)
36+
37+
testCases := []struct {
38+
name string
39+
coreProvider client.Object
40+
expected []ctrl.Request
41+
}{
42+
{
43+
name: "Core provider Ready condition is True",
44+
coreProvider: &operatorv1.CoreProvider{
45+
ObjectMeta: metav1.ObjectMeta{
46+
Name: "core-provider",
47+
Namespace: testNamespaceName,
48+
},
49+
Status: operatorv1.CoreProviderStatus{
50+
ProviderStatus: operatorv1.ProviderStatus{
51+
Conditions: clusterv1.Conditions{
52+
{
53+
Type: clusterv1.ReadyCondition,
54+
Status: corev1.ConditionTrue,
55+
LastTransitionTime: metav1.Now(),
56+
Message: "Provider is ready",
57+
},
58+
},
59+
},
60+
},
61+
},
62+
expected: []reconcile.Request{
63+
{NamespacedName: types.NamespacedName{Namespace: testNamespaceName, Name: "preflight-checks-condition-false"}},
64+
{NamespacedName: types.NamespacedName{Namespace: testNamespaceName, Name: "empty-status-conditions"}},
65+
},
66+
},
67+
{
68+
name: "Core provider is not ready",
69+
coreProvider: &operatorv1.CoreProvider{
70+
ObjectMeta: metav1.ObjectMeta{
71+
Name: "core-provider",
72+
Namespace: testNamespaceName,
73+
},
74+
},
75+
expected: []reconcile.Request{},
76+
},
77+
}
78+
k8sClient := fake.NewClientBuilder().
79+
WithScheme(setupScheme()).
80+
WithObjects(
81+
&operatorv1.InfrastructureProvider{
82+
ObjectMeta: metav1.ObjectMeta{
83+
Name: "preflight-checks-condition-false",
84+
Namespace: testNamespaceName,
85+
},
86+
Spec: operatorv1.InfrastructureProviderSpec{
87+
ProviderSpec: operatorv1.ProviderSpec{},
88+
},
89+
Status: operatorv1.InfrastructureProviderStatus{
90+
ProviderStatus: operatorv1.ProviderStatus{
91+
Conditions: clusterv1.Conditions{
92+
{
93+
Type: operatorv1.PreflightCheckCondition,
94+
Status: corev1.ConditionFalse,
95+
LastTransitionTime: metav1.Now(),
96+
Reason: operatorv1.WaitingForCoreProviderReadyReason,
97+
Message: "Core provider is not ready",
98+
},
99+
},
100+
},
101+
},
102+
},
103+
&operatorv1.InfrastructureProvider{
104+
ObjectMeta: metav1.ObjectMeta{
105+
Name: "preflight-checks-condition-true",
106+
Namespace: testNamespaceName,
107+
},
108+
Spec: operatorv1.InfrastructureProviderSpec{
109+
ProviderSpec: operatorv1.ProviderSpec{},
110+
},
111+
Status: operatorv1.InfrastructureProviderStatus{
112+
ProviderStatus: operatorv1.ProviderStatus{
113+
Conditions: clusterv1.Conditions{
114+
{
115+
Type: operatorv1.PreflightCheckCondition,
116+
Status: corev1.ConditionTrue,
117+
LastTransitionTime: metav1.Now(),
118+
Message: "Core provider is ready",
119+
},
120+
},
121+
},
122+
},
123+
},
124+
&operatorv1.InfrastructureProvider{
125+
ObjectMeta: metav1.ObjectMeta{
126+
Name: "empty-status-conditions",
127+
Namespace: testNamespaceName,
128+
},
129+
Spec: operatorv1.InfrastructureProviderSpec{
130+
ProviderSpec: operatorv1.ProviderSpec{},
131+
},
132+
},
133+
).
134+
Build()
135+
136+
for _, tc := range testCases {
137+
t.Run(tc.name, func(t *testing.T) {
138+
requests := newCoreProviderToProviderFuncMapForProviderList(k8sClient, &operatorv1.InfrastructureProviderList{})(ctx, tc.coreProvider)
139+
g.Expect(requests).To(HaveLen(len(tc.expected)))
140+
g.Expect(requests).To(ContainElements(tc.expected))
141+
})
142+
}
143+
}

‎internal/controller/genericprovider_controller.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"errors"
2424
"fmt"
2525
"hash"
26+
"reflect"
2627

2728
corev1 "k8s.io/api/core/v1"
2829
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -54,16 +55,32 @@ const (
5455
appliedSpecHashAnnotation = "operator.cluster.x-k8s.io/applied-spec-hash"
5556
)
5657

57-
func (r *GenericProviderReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
58+
func (r *GenericProviderReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
5859
builder := ctrl.NewControllerManagedBy(mgr).
5960
For(r.Provider)
6061
if r.WatchConfigSecretChanges {
62+
if err := mgr.GetFieldIndexer().IndexField(ctx, r.Provider, configSecretNameField, configSecretNameIndexFunc); err != nil {
63+
return err
64+
}
65+
66+
if err := mgr.GetFieldIndexer().IndexField(ctx, r.Provider, configSecretNamespaceField, configSecretNamespaceIndexFunc); err != nil {
67+
return err
68+
}
69+
6170
builder.Watches(
6271
&corev1.Secret{},
6372
handler.EnqueueRequestsFromMapFunc(newSecretToProviderFuncMapForProviderList(r.Client, r.ProviderList)),
6473
)
6574
}
6675

76+
// We don't want to receive secondary events from the CoreProvider for itself.
77+
if reflect.TypeOf(r.Provider) != reflect.TypeOf(genericprovider.GenericProvider(&operatorv1.CoreProvider{})) {
78+
builder.Watches(
79+
&operatorv1.CoreProvider{},
80+
handler.EnqueueRequestsFromMapFunc(newCoreProviderToProviderFuncMapForProviderList(r.Client, r.ProviderList)),
81+
)
82+
}
83+
6784
return builder.WithOptions(options).
6885
Complete(r)
6986
}

0 commit comments

Comments
 (0)