Skip to content

Commit 94fd5cd

Browse files
authored
Merge pull request #6495 from fabriziopandini/topology-ssa
✨ Add Server Side Apply helper to the topology controller
2 parents 88dc60e + 2f6f77f commit 94fd5cd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+4374
-3006
lines changed

api/v1beta1/common_types.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const (
4040
// to easily discover which fields have been set by templates + patches/variables at a given reconcile;
4141
// instead, it is not necessary to store managed paths for typed objets (e.g. Cluster, MachineDeployments)
4242
// given that the topology controller explicitly sets a well-known, immutable list of fields at every reconcile.
43+
// Deprecated: Topology controller is now using server side apply and this annotation will be removed in a future release.
4344
ClusterTopologyManagedFieldsAnnotation = "topology.cluster.x-k8s.io/managed-field-paths"
4445

4546
// ClusterTopologyMachineDeploymentLabelName is the label set on the generated MachineDeployment objects

cmd/clusterctl/client/cluster/client.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,6 @@ import (
2929
logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
3030
)
3131

32-
const (
33-
minimumKubernetesVersion = "v1.20.0"
34-
)
35-
3632
var (
3733
ctx = context.TODO()
3834
)

cmd/clusterctl/client/cluster/mover.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package cluster
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"os"
2223
"path/filepath"
@@ -34,6 +35,7 @@ import (
3435

3536
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3637
logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
38+
"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/structuredmerge"
3739
"sigs.k8s.io/cluster-api/util/annotations"
3840
"sigs.k8s.io/cluster-api/util/conditions"
3941
"sigs.k8s.io/cluster-api/util/patch"
@@ -851,6 +853,9 @@ func (o *objectMover) createTargetObject(nodeToCreate *node, toProxy Proxy) erro
851853
// Rebuild the owne reference chain
852854
o.buildOwnerChain(obj, nodeToCreate)
853855

856+
// Save the old managed fields for topology managed fields migration
857+
oldManagedFields := obj.GetManagedFields()
858+
854859
// FIXME Workaround for https://github.com/kubernetes/kubernetes/issues/32220. Remove when the issue is fixed.
855860
// If the resource already exists, the API server ordinarily returns an AlreadyExists error. Due to the above issue, if the resource has a non-empty metadata.generateName field, the API server returns a ServerTimeoutError. To ensure that the API server returns an AlreadyExists error, we set the metadata.generateName field to an empty string.
856861
if len(obj.GetName()) > 0 && len(obj.GetGenerateName()) > 0 {
@@ -897,6 +902,10 @@ func (o *objectMover) createTargetObject(nodeToCreate *node, toProxy Proxy) erro
897902
// Stores the newUID assigned to the newly created object.
898903
nodeToCreate.newUID = obj.GetUID()
899904

905+
if err := patchTopologyManagedFields(ctx, oldManagedFields, obj, cTo); err != nil {
906+
return err
907+
}
908+
900909
return nil
901910
}
902911

@@ -1164,3 +1173,32 @@ func (o *objectMover) checkTargetProviders(toInventory InventoryClient) error {
11641173

11651174
return kerrors.NewAggregate(errList)
11661175
}
1176+
1177+
// patchTopologyManagedFields patches the managed fields of obj if parts of it are owned by the topology controller.
1178+
// This is necessary to ensure the managed fields created by the topology controller are still present and thus to
1179+
// prevent unnecessary machine rollouts. Without patching the managed fields, clusterctl would be the owner of the fields
1180+
// which would lead to co-ownership and also additional machine rollouts.
1181+
func patchTopologyManagedFields(ctx context.Context, oldManagedFields []metav1.ManagedFieldsEntry, obj *unstructured.Unstructured, cTo client.Client) error {
1182+
var containsTopologyManagedFields bool
1183+
// Check if the object was owned by the topology controller.
1184+
for _, m := range oldManagedFields {
1185+
if m.Operation == metav1.ManagedFieldsOperationApply &&
1186+
m.Manager == structuredmerge.TopologyManagerName &&
1187+
m.Subresource == "" {
1188+
containsTopologyManagedFields = true
1189+
break
1190+
}
1191+
}
1192+
// Return early if the object was not owned by the topology controller.
1193+
if !containsTopologyManagedFields {
1194+
return nil
1195+
}
1196+
base := obj.DeepCopy()
1197+
obj.SetManagedFields(oldManagedFields)
1198+
1199+
if err := cTo.Patch(ctx, obj, client.MergeFrom(base)); err != nil {
1200+
return errors.Wrapf(err, "error patching managed fields %q %s/%s",
1201+
obj.GroupVersionKind(), obj.GetNamespace(), obj.GetName())
1202+
}
1203+
return nil
1204+
}

cmd/clusterctl/client/cluster/proxy.go

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ package cluster
1818

1919
import (
2020
"fmt"
21+
"os"
22+
"strconv"
2123
"strings"
2224
"time"
2325

@@ -27,7 +29,6 @@ import (
2729
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2830
"k8s.io/apimachinery/pkg/runtime/schema"
2931
"k8s.io/apimachinery/pkg/util/sets"
30-
utilversion "k8s.io/apimachinery/pkg/util/version"
3132
"k8s.io/client-go/discovery"
3233
"k8s.io/client-go/kubernetes"
3334
"k8s.io/client-go/rest"
@@ -52,7 +53,7 @@ type Proxy interface {
5253
// CurrentNamespace returns the namespace from the current context in the kubeconfig file.
5354
CurrentNamespace() (string, error)
5455

55-
// ValidateKubernetesVersion returns an error if management cluster version less than minimumKubernetesVersion.
56+
// ValidateKubernetesVersion returns an error if management cluster version less than MinimumKubernetesVersion.
5657
ValidateKubernetesVersion() error
5758

5859
// NewClient returns a new controller runtime Client object for working on the management cluster.
@@ -119,22 +120,12 @@ func (k *proxy) ValidateKubernetesVersion() error {
119120
return err
120121
}
121122

122-
client := discovery.NewDiscoveryClientForConfigOrDie(config)
123-
serverVersion, err := client.ServerVersion()
124-
if err != nil {
125-
return errors.Wrap(err, "failed to retrieve server version")
126-
}
127-
128-
compver, err := utilversion.MustParseGeneric(serverVersion.String()).Compare(minimumKubernetesVersion)
129-
if err != nil {
130-
return errors.Wrap(err, "failed to parse and compare server version")
123+
minVer := version.MinimumKubernetesVersion
124+
if clusterTopologyFeatureGate, _ := strconv.ParseBool(os.Getenv("CLUSTER_TOPOLOGY")); clusterTopologyFeatureGate {
125+
minVer = version.MinimumKubernetesVersionClusterTopology
131126
}
132127

133-
if compver == -1 {
134-
return errors.Errorf("unsupported management cluster server version: %s - minimum required version is %s", serverVersion.String(), minimumKubernetesVersion)
135-
}
136-
137-
return nil
128+
return version.CheckKubernetesVersion(config, minVer)
138129
}
139130

140131
// GetConfig returns the config for a kubernetes client.

docs/book/src/clusterctl/commands/alpha-topology-plan.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,27 @@ the input should have all the objects needed.
2222

2323
</aside>
2424

25+
<aside class="note">
26+
27+
<h1>Limitations</h1>
28+
29+
The topology controllers uses [Server Side Apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/)
30+
to support use cases where other controllers are co-authoring the same objects, but this kind of interactions can't be recreated
31+
in a dry-run scenario.
32+
33+
As a consequence Dry-Run can give some false positives/false negatives when trying to have a preview of
34+
changes to a set of existing topology owned objects. In other worlds this limitation impacts all the use cases described
35+
below except for "Designing a new ClusterClass".
36+
37+
More specifically:
38+
- DryRun doesn't consider OpenAPI schema extension like +ListMap this can lead to false positives when topology
39+
dry run is simulating a change to an existing slice (DryRun always reverts external changes, like server side apply when +ListMap=atomic).
40+
- DryRun doesn't consider existing metadata.managedFields, and this can lead to false negatives when topology dry run
41+
is simulating a change where a field is dropped from a template (DryRun always preserve dropped fields, like
42+
server side apply when the field has more than one manager).
43+
44+
</aside>
45+
2546
## Example use cases
2647

2748
### Designing a new ClusterClass

docs/book/src/developer/providers/v1.1-to-v1.2.md

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ maintainers of providers and consumers of our Go API.
55

66
## Minimum Kubernetes version for the management cluster
77

8-
* The minimum Kubernetes version that can be used for a management cluster by Cluster API is now 1.20.0
8+
* The minimum Kubernetes version that can be used for a management cluster is now 1.20.0
9+
* The minimum Kubernetes version that can be used for a management cluster with ClusterClass is now 1.22.0
10+
11+
NOTE: compliance with minimum Kubernetes version is enforced both by clusterctl and when the CAPI controller starts.
912

1013
## Minimum Go version
1114

@@ -28,6 +31,7 @@ in ClusterAPI are kept in sync with the versions used by `sigs.k8s.io/controller
2831
### Deprecation
2932

3033
* `util.MachinesByCreationTimestamp` has been deprecated and will be removed in a future release.
34+
* the `topology.cluster.x-k8s.io/managed-field-paths` annotation has been deprecated and it will be removed in a future release.
3135

3236
### Removals
3337
* The `third_party/kubernetes-drain` package has been removed, as we're now using `k8s.io/kubectl/pkg/drain` instead ([PR](https://github.com/kubernetes-sigs/cluster-api/pull/5440)).
@@ -36,11 +40,34 @@ in ClusterAPI are kept in sync with the versions used by `sigs.k8s.io/controller
3640
`annotations.HasPaused` and `annotations.HasSkipRemediation` respectively instead.
3741
* `ObjectMeta.ClusterName` has been removed from `k8s.io/apimachinery/pkg/apis/meta/v1`.
3842

39-
### API Changes
43+
### golang API Changes
4044

4145
- `util.ClusterToInfrastructureMapFuncWithExternallyManagedCheck` was removed and the externally managed check was added to `util.ClusterToInfrastructureMapFunc`, which required changing its signature.
4246
Users of the former simply need to start using the latter and users of the latter need to add the new arguments to their call.
4347

48+
### Required API Changes for providers
49+
50+
- ClusterClass and managed topologies are now using [Server Side Apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/)
51+
to properly manage other controllers like CAPA/CAPZ coauthoring slices, see [#6320](https://github.com/kubernetes-sigs/cluster-api/issues/6320).
52+
In order to take advantage of this feature providers are required to add marker to their API types as described in
53+
[merge-strategy](https://kubernetes.io/docs/reference/using-api/server-side-apply/#merge-strategy).
54+
NOTE: the change will cause a rollout on existing clusters created with ClusterClass
55+
56+
E.g. in CAPA
57+
58+
```go
59+
// +optional
60+
Subnets Subnets `json:"subnets,omitempty"
61+
```
62+
Must be modified into:
63+
64+
```go
65+
// +optional
66+
// +listType=map
67+
// +listMapKey=id
68+
Subnets Subnets `json:"subnets,omitempty"
69+
```
70+
4471
### Other
4572

4673
- Logging:

docs/book/src/reference/versions.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ These diagrams show the relationships between components in a Cluster API releas
7777

7878
\* There is an issue with CRDs in Kubernetes v1.23.{0-2}. ClusterClass with patches is affected by that (for more details please see [this issue](https://github.com/kubernetes-sigs/cluster-api/issues/5990)). Therefore we recommend to use Kubernetes v1.23.3+ with ClusterClass.
7979
Previous Kubernetes **minor** versions are not affected.
80+
\** When using CAPI v1.2 with the CLUSTER_TOPOLOGY experimental feature on, the Kubernetes Version for the management cluster must be >= 1.22.0.
8081

8182
The Core Provider also talks to API server of every Workload Cluster. Therefore, the Workload Cluster's Kubernetes version must also be compatible.
8283

docs/book/src/tasks/experimental-features/cluster-class/change-clusterclass.md

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -174,21 +174,19 @@ underlying objects like control plane and MachineDeployment act in the same way
174174
The topology reconciler enforces values defined in the ClusterClass templates into the topology
175175
owned objects in a Cluster.
176176

177-
A simple way to understand this is to `kubectl get -o json` templates referenced in a ClusterClass;
178-
then you can consider the topology reconciler to be authoritative on all the values
179-
under `spec`. Being authoritative means that the user cannot manually change those values in
180-
the object derived from the template in a specific Cluster (and if they do so the value gets reconciled
181-
to the value defined in the ClusterClass).
177+
More specifically, the topology controller uses [Server Side Apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/)
178+
to write/patch topology owned objects; using SSA allows other controllers to co-author the generated objects,
179+
like e.g. adding info for subnets in CAPA.
182180

183181
<aside class="note">
184182
<h1>What about patches?</h1>
185183

186184
The considerations above apply also when using patches, the only difference being that the
187-
authoritative fields should be determined by applying patches on top of the `kubectl get -o json` output.
185+
set of fields that are enforced should be determined by applying patches on top of the templates.
188186

189187
</aside>
190188

191-
A corollary of the behaviour described above is that it is technically possible to change non-authoritative
192-
fields in the object derived from the template in a specific Cluster, but we advise against using the possibility
189+
A corollary of the behaviour described above is that it is technically possible to change fields in the object
190+
which are not derived from the templates and patches, but we advise against using the possibility
193191
or making ad-hoc changes in generated objects unless otherwise needed for a workaround. It is always
194192
preferable to improve ClusterClasses by supporting new Cluster variants in a reusable way.

docs/book/src/tasks/experimental-features/cluster-class/index.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,14 @@
33
The ClusterClass feature introduces a new way to create clusters which reduces boilerplate and enables flexible and powerful customization of clusters.
44
ClusterClass is a powerful abstraction implemented on top of existing interfaces and offers a set of tools and operations to streamline cluster lifecycle management while maintaining the same underlying API.
55

6+
</aside>
7+
8+
<aside class="note warning">
9+
10+
In order to use the ClusterClass (alpha) experimental feature the Kubernetes Version for the management cluster must be >= 1.22.0.
11+
12+
</aside>
13+
614
**Feature gate name**: `ClusterTopology`
715

816
**Variable name to enable/disable the feature gate**: `CLUSTER_TOPOLOGY`

internal/contract/types.go

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,47 @@ var errNotFound = errors.New("not found")
2929
// Path defines a how to access a field in an Unstructured object.
3030
type Path []string
3131

32+
// Append a field name to a path.
33+
func (p Path) Append(k string) Path {
34+
return append(p, k)
35+
}
36+
37+
// IsParentOf check if one path is Parent of the other.
38+
func (p Path) IsParentOf(other Path) bool {
39+
if len(p) >= len(other) {
40+
return false
41+
}
42+
for i := range p {
43+
if p[i] != other[i] {
44+
return false
45+
}
46+
}
47+
return true
48+
}
49+
50+
// Equal check if two path are equal (exact match).
51+
func (p Path) Equal(other Path) bool {
52+
if len(p) != len(other) {
53+
return false
54+
}
55+
for i := range p {
56+
if p[i] != other[i] {
57+
return false
58+
}
59+
}
60+
return true
61+
}
62+
63+
// Overlaps return true if two paths are Equal or one IsParentOf the other.
64+
func (p Path) Overlaps(other Path) bool {
65+
return other.Equal(p) || other.IsParentOf(p) || p.IsParentOf(other)
66+
}
67+
68+
// String returns the path as a dotted string.
69+
func (p Path) String() string {
70+
return strings.Join(p, ".")
71+
}
72+
3273
// Int64 represents an accessor to an int64 path value.
3374
type Int64 struct {
3475
path Path

0 commit comments

Comments
 (0)