Skip to content

Commit 4fc113e

Browse files
dmvoloddmvolod
committed
Init helm release drift manager POC based on dynamic manifest objects watch
1 parent 80af83d commit 4fc113e

15 files changed

+642
-15
lines changed

‎api/v1alpha1/helmchartproxy_types.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ type HelmChartProxySpec struct {
8484
// +optional
8585
ReconcileStrategy string `json:"reconcileStrategy,omitempty"`
8686

87+
ReleaseDrift bool `json:"releaseDrift,omitempty"`
88+
8789
// Options represents CLI flags passed to Helm operations (i.e. install, upgrade, delete) and
8890
// include options such as wait, skipCRDs, timeout, waitForJobs, etc.
8991
// +optional

‎api/v1alpha1/helmreleaseproxy_types.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ type HelmReleaseProxySpec struct {
7878
// +optional
7979
ReconcileStrategy string `json:"reconcileStrategy,omitempty"`
8080

81+
ReleaseDrift bool `json:"releaseDrift,omitempty"`
82+
8183
// Options represents the helm setting options which can be used to control behaviour of helm operations(Install, Upgrade, Delete, etc)
8284
// via options like wait, skipCrds, timeout, waitForJobs, etc.
8385
// +optional

‎config/crd/bases/addons.cluster.x-k8s.io_helmchartproxies.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,8 @@ spec:
270270
- InstallOnce
271271
- Continuous
272272
type: string
273+
releaseDrift:
274+
type: boolean
273275
releaseName:
274276
description: ReleaseName is the release name of the installed Helm
275277
chart. If it is not specified, a name will be generated.

‎config/crd/bases/addons.cluster.x-k8s.io_helmreleaseproxies.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,8 @@ spec:
277277
- InstallOnce
278278
- Continuous
279279
type: string
280+
releaseDrift:
281+
type: boolean
280282
releaseName:
281283
description: ReleaseName is the release name of the installed Helm
282284
chart. If it is not specified, a name will be generated.

‎controllers/helmchartproxy/helmchartproxy_controller_phases.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ func constructHelmReleaseProxy(existing *addonsv1alpha1.HelmReleaseProxy, helmCh
227227
}
228228

229229
helmReleaseProxy.Spec.ReconcileStrategy = helmChartProxy.Spec.ReconcileStrategy
230+
helmReleaseProxy.Spec.ReleaseDrift = helmChartProxy.Spec.ReleaseDrift
230231
helmReleaseProxy.Spec.Version = helmChartProxy.Spec.Version
231232
helmReleaseProxy.Spec.Values = parsedValues
232233
helmReleaseProxy.Spec.Options = helmChartProxy.Spec.Options

‎controllers/helmreleasedrift/manager.go

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package helmreleasedrift
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
"sync"
24+
25+
"github.com/ironcore-dev/controller-utils/unstructuredutils"
26+
"golang.org/x/exp/slices"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/apimachinery/pkg/labels"
29+
"k8s.io/apimachinery/pkg/runtime/schema"
30+
"k8s.io/client-go/kubernetes/scheme"
31+
"k8s.io/client-go/rest"
32+
addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
33+
ctrl "sigs.k8s.io/controller-runtime"
34+
"sigs.k8s.io/controller-runtime/pkg/cache"
35+
"sigs.k8s.io/controller-runtime/pkg/client"
36+
"sigs.k8s.io/controller-runtime/pkg/event"
37+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
38+
"sigs.k8s.io/kustomize/api/konfig"
39+
)
40+
41+
const (
42+
InstanceLabelKey = "app.kubernetes.io/instance"
43+
)
44+
45+
var (
46+
managers = map[string]options{}
47+
mutex sync.Mutex
48+
)
49+
50+
type options struct {
51+
gvks []schema.GroupVersionKind
52+
cancel context.CancelFunc
53+
}
54+
55+
func Add(ctx context.Context, restConfig *rest.Config, helmReleaseProxy *addonsv1alpha1.HelmReleaseProxy, releaseManifest string, eventChannel chan event.GenericEvent) error {
56+
log := ctrl.LoggerFrom(ctx)
57+
gvks, err := extractGVKsFromManifest(releaseManifest)
58+
if err != nil {
59+
return err
60+
}
61+
62+
manager, exist := managers[managerKey(helmReleaseProxy)]
63+
if exist {
64+
if slices.Equal(manager.gvks, gvks) {
65+
return nil
66+
}
67+
Remove(helmReleaseProxy)
68+
}
69+
70+
mutex.Lock()
71+
defer mutex.Unlock()
72+
k8sManager, err := ctrl.NewManager(restConfig, ctrl.Options{
73+
Scheme: scheme.Scheme,
74+
Metrics: metricsserver.Options{
75+
BindAddress: "0",
76+
},
77+
HealthProbeBindAddress: "0",
78+
Cache: cache.Options{
79+
DefaultLabelSelector: labels.SelectorFromSet(map[string]string{
80+
konfig.ManagedbyLabelKey: "Helm",
81+
InstanceLabelKey: helmReleaseProxy.Spec.ReleaseName,
82+
}),
83+
},
84+
})
85+
if err != nil {
86+
return err
87+
}
88+
if err = (&releaseDriftReconciler{
89+
Client: k8sManager.GetClient(),
90+
Scheme: k8sManager.GetScheme(),
91+
HelmReleaseProxyKey: client.ObjectKeyFromObject(helmReleaseProxy),
92+
HelmReleaseProxyEvent: eventChannel,
93+
}).setupWithManager(k8sManager, gvks); err != nil {
94+
return err
95+
}
96+
log.V(2).Info("Starting release drift controller manager")
97+
ctx, cancel := context.WithCancel(ctx)
98+
go func() {
99+
if err = k8sManager.Start(ctx); err != nil {
100+
log.V(2).Error(err, "failed to start release drift manager")
101+
objectMeta := metav1.ObjectMeta{
102+
Name: helmReleaseProxy.Name,
103+
Namespace: helmReleaseProxy.Namespace,
104+
}
105+
eventChannel <- event.GenericEvent{Object: &addonsv1alpha1.HelmReleaseProxy{ObjectMeta: objectMeta}}
106+
}
107+
}()
108+
109+
managers[managerKey(helmReleaseProxy)] = options{
110+
gvks: gvks,
111+
cancel: cancel,
112+
}
113+
114+
return nil
115+
}
116+
117+
func Remove(helmReleaseProxy *addonsv1alpha1.HelmReleaseProxy) {
118+
mutex.Lock()
119+
defer mutex.Unlock()
120+
121+
manager, exist := managers[managerKey(helmReleaseProxy)]
122+
if exist {
123+
manager.cancel()
124+
delete(managers, managerKey(helmReleaseProxy))
125+
}
126+
}
127+
128+
func managerKey(helmReleaseProxy *addonsv1alpha1.HelmReleaseProxy) string {
129+
return fmt.Sprintf("%s-%s-%s", helmReleaseProxy.Spec.ClusterRef.Name, helmReleaseProxy.Namespace, helmReleaseProxy.Spec.ReleaseName)
130+
}
131+
132+
func extractGVKsFromManifest(manifest string) ([]schema.GroupVersionKind, error) {
133+
objects, err := unstructuredutils.Read(strings.NewReader(manifest))
134+
if err != nil {
135+
return nil, err
136+
}
137+
var gvks []schema.GroupVersionKind
138+
for _, obj := range objects {
139+
if !slices.Contains(gvks, obj.GroupVersionKind()) {
140+
gvks = append(gvks, obj.GroupVersionKind())
141+
}
142+
}
143+
144+
return gvks, nil
145+
}

‎controllers/helmreleasedrift/manager_test.go

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package helmreleasedrift_test
18+
19+
import (
20+
"github.com/ironcore-dev/controller-utils/metautils"
21+
. "github.com/onsi/ginkgo/v2"
22+
. "github.com/onsi/gomega"
23+
appsv1 "k8s.io/api/apps/v1"
24+
corev1 "k8s.io/api/core/v1"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"k8s.io/utils/ptr"
27+
addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
28+
"sigs.k8s.io/cluster-api-addon-provider-helm/controllers/helmreleasedrift"
29+
"sigs.k8s.io/cluster-api-addon-provider-helm/controllers/helmreleasedrift/test/fake"
30+
"sigs.k8s.io/controller-runtime/pkg/client"
31+
"sigs.k8s.io/controller-runtime/pkg/event"
32+
)
33+
34+
const (
35+
releaseName = "ahoy"
36+
objectName = "ahoy-hello-world"
37+
originalDeploymentReplicas = 1
38+
patchedDeploymentReplicas = 3
39+
)
40+
41+
var _ = Describe("Testing HelmReleaseProxy drift manager with fake manifest", func() {
42+
It("Adding HelmReleaseProxy drift manager and validating its lifecycle", func() {
43+
objectMeta := metav1.ObjectMeta{
44+
Name: releaseName,
45+
Namespace: metav1.NamespaceDefault,
46+
}
47+
fake.ManifestEventChannel <- event.GenericEvent{Object: &addonsv1alpha1.HelmReleaseProxy{ObjectMeta: objectMeta}}
48+
49+
helmReleaseProxy := &addonsv1alpha1.HelmReleaseProxy{
50+
ObjectMeta: metav1.ObjectMeta{
51+
Name: "ahoy-release-proxy",
52+
Namespace: metav1.NamespaceDefault,
53+
},
54+
Spec: addonsv1alpha1.HelmReleaseProxySpec{
55+
ReleaseName: releaseName,
56+
},
57+
}
58+
59+
// TODO (dvolodin) Find way how to wait manager to start for testing
60+
err := helmreleasedrift.Add(ctx, restConfig, helmReleaseProxy, manifest, fake.ManifestEventChannel)
61+
Expect(err).NotTo(HaveOccurred())
62+
63+
Eventually(func() bool {
64+
for _, objectList := range []client.ObjectList{&corev1.ServiceList{}, &appsv1.DeploymentList{}, &corev1.ServiceAccountList{}} {
65+
err := k8sClient.List(ctx, objectList, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels(map[string]string{helmreleasedrift.InstanceLabelKey: releaseName}))
66+
if err != nil {
67+
return false
68+
}
69+
objects, err := metautils.ExtractList(objectList)
70+
if err != nil || len(objects) == 0 {
71+
return false
72+
}
73+
}
74+
75+
return true
76+
}, timeout, interval).Should(BeTrue())
77+
78+
deployment := &appsv1.Deployment{}
79+
err = k8sClient.Get(ctx, client.ObjectKey{Name: objectName, Namespace: metav1.NamespaceDefault}, deployment)
80+
Expect(err).NotTo(HaveOccurred())
81+
patch := client.MergeFrom(deployment.DeepCopy())
82+
deployment.Spec.Replicas = ptr.To(int32(patchedDeploymentReplicas))
83+
err = k8sClient.Patch(ctx, deployment, patch)
84+
Expect(err).NotTo(HaveOccurred())
85+
86+
Eventually(func() bool {
87+
err = k8sClient.Get(ctx, client.ObjectKey{Name: objectName, Namespace: metav1.NamespaceDefault}, deployment)
88+
return err == nil && *deployment.Spec.Replicas == originalDeploymentReplicas
89+
}, timeout, interval).Should(BeTrue())
90+
91+
helmreleasedrift.Remove(helmReleaseProxy)
92+
})
93+
})

‎controllers/helmreleasedrift/releasedrift_controller.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package helmreleasedrift
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
25+
"k8s.io/apimachinery/pkg/runtime"
26+
"k8s.io/apimachinery/pkg/runtime/schema"
27+
addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
28+
ctrl "sigs.k8s.io/controller-runtime"
29+
"sigs.k8s.io/controller-runtime/pkg/builder"
30+
"sigs.k8s.io/controller-runtime/pkg/client"
31+
"sigs.k8s.io/controller-runtime/pkg/event"
32+
"sigs.k8s.io/controller-runtime/pkg/handler"
33+
"sigs.k8s.io/controller-runtime/pkg/predicate"
34+
)
35+
36+
// releaseDriftReconciler reconciles an event from the all helm objects managed by the HelmReleaseProxy.
37+
type releaseDriftReconciler struct {
38+
client.Client
39+
Scheme *runtime.Scheme
40+
HelmReleaseProxyKey client.ObjectKey
41+
HelmReleaseProxyEvent chan event.GenericEvent
42+
}
43+
44+
var excludeCreateEventsPredicate = predicate.Funcs{
45+
CreateFunc: func(e event.CreateEvent) bool {
46+
return false
47+
},
48+
}
49+
50+
// setupWithManager sets up the controller with the Manager.
51+
func (r *releaseDriftReconciler) setupWithManager(mgr ctrl.Manager, gvks []schema.GroupVersionKind) error {
52+
controllerBuilder := ctrl.NewControllerManagedBy(mgr).
53+
Named(fmt.Sprintf("%s-%s-release-drift-controller", r.HelmReleaseProxyKey.Name, r.HelmReleaseProxyKey.Namespace))
54+
for _, gvk := range gvks {
55+
watch := &unstructured.Unstructured{}
56+
watch.SetGroupVersionKind(gvk)
57+
controllerBuilder.Watches(watch, handler.EnqueueRequestsFromMapFunc(r.WatchesToReleaseMapper), builder.OnlyMetadata)
58+
}
59+
60+
return controllerBuilder.WithEventFilter(excludeCreateEventsPredicate).Complete(r)
61+
}
62+
63+
// Reconcile is part of the main kubernetes reconciliation loop which aims to
64+
// move the current state of the cluster closer to the desired state.
65+
func (r *releaseDriftReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {
66+
log := ctrl.LoggerFrom(ctx)
67+
log.V(2).Info("Beginning reconciliation", "requestNamespace", req.Namespace, "requestName", req.Name)
68+
69+
objectMeta := metav1.ObjectMeta{
70+
Name: r.HelmReleaseProxyKey.Name,
71+
Namespace: r.HelmReleaseProxyKey.Namespace,
72+
}
73+
r.HelmReleaseProxyEvent <- event.GenericEvent{Object: &addonsv1alpha1.HelmReleaseProxy{ObjectMeta: objectMeta}}
74+
75+
return ctrl.Result{}, nil
76+
}
77+
78+
func (r *releaseDriftReconciler) WatchesToReleaseMapper(_ context.Context, _ client.Object) []ctrl.Request {
79+
return []ctrl.Request{{NamespacedName: r.HelmReleaseProxyKey}}
80+
}

0 commit comments

Comments
 (0)