Skip to content

Commit 3cfb41d

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #11520 from AndiDog/recreate-bootstrap-token-if-cleaned
🐛 Recreate bootstrap token if it was cleaned up
2 parents c9261b0 + 51a4011 commit 3cfb41d

File tree

2 files changed

+177
-12
lines changed

2 files changed

+177
-12
lines changed

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go

+28-12
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, c
320320
// If the BootstrapToken has been generated for a join but the config owner has no nodeRefs,
321321
// this indicates that the node has not yet joined and the token in the join config has not
322322
// been consumed and it may need a refresh.
323-
return r.refreshBootstrapTokenIfNeeded(ctx, config, cluster)
323+
return r.refreshBootstrapTokenIfNeeded(ctx, config, cluster, scope)
324324
}
325325
if configOwner.IsMachinePool() {
326326
// If the BootstrapToken has been generated and infrastructure is ready but the configOwner is a MachinePool,
@@ -358,7 +358,7 @@ func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, c
358358
return r.joinWorker(ctx, scope)
359359
}
360360

361-
func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster) (ctrl.Result, error) {
361+
func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
362362
log := ctrl.LoggerFrom(ctx)
363363
token := config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
364364

@@ -369,6 +369,11 @@ func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Cont
369369

370370
secret, err := getToken(ctx, remoteClient, token)
371371
if err != nil {
372+
if apierrors.IsNotFound(err) && scope.ConfigOwner.IsMachinePool() {
373+
log.Info("Bootstrap token secret not found, triggering creation of new token")
374+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = ""
375+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
376+
}
372377
return ctrl.Result{}, errors.Wrapf(err, "failed to get bootstrap token secret in order to refresh it")
373378
}
374379
log = log.WithValues("Secret", klog.KObj(secret))
@@ -399,13 +404,33 @@ func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Cont
399404
log.Info("Refreshing token until the infrastructure has a chance to consume it", "oldExpiration", secretExpiration, "newExpiration", newExpiration)
400405
err = remoteClient.Update(ctx, secret)
401406
if err != nil {
407+
if apierrors.IsNotFound(err) && scope.ConfigOwner.IsMachinePool() {
408+
log.Info("Bootstrap token secret not found, triggering creation of new token")
409+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = ""
410+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
411+
}
402412
return ctrl.Result{}, errors.Wrapf(err, "failed to refresh bootstrap token")
403413
}
404414
return ctrl.Result{
405415
RequeueAfter: r.tokenCheckRefreshOrRotationInterval(),
406416
}, nil
407417
}
408418

419+
func (r *KubeadmConfigReconciler) recreateBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, scope *Scope, remoteClient client.Client) (ctrl.Result, error) {
420+
log := ctrl.LoggerFrom(ctx)
421+
422+
token, err := createToken(ctx, remoteClient, r.TokenTTL)
423+
if err != nil {
424+
return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
425+
}
426+
427+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
428+
log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
429+
430+
// Update the bootstrap data
431+
return r.joinWorker(ctx, scope)
432+
}
433+
409434
func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
410435
log := ctrl.LoggerFrom(ctx)
411436
log.V(2).Info("Config is owned by a MachinePool, checking if token should be rotated")
@@ -421,16 +446,7 @@ func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Co
421446
}
422447
if shouldRotate {
423448
log.Info("Creating new bootstrap token, the existing one should be rotated")
424-
token, err := createToken(ctx, remoteClient, r.TokenTTL)
425-
if err != nil {
426-
return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
427-
}
428-
429-
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
430-
log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
431-
432-
// update the bootstrap data
433-
return r.joinWorker(ctx, scope)
449+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
434450
}
435451
return ctrl.Result{
436452
RequeueAfter: r.tokenCheckRefreshOrRotationInterval(),

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go

+149
Original file line numberDiff line numberDiff line change
@@ -1441,6 +1441,155 @@ func TestBootstrapTokenRotationMachinePool(t *testing.T) {
14411441
g.Expect(foundNew).To(BeTrue())
14421442
}
14431443

1444+
func TestBootstrapTokenRefreshIfTokenSecretCleaned(t *testing.T) {
1445+
t.Run("should not recreate the token for Machines", func(t *testing.T) {
1446+
g := NewWithT(t)
1447+
1448+
cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
1449+
cluster.Status.InfrastructureReady = true
1450+
conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
1451+
cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
1452+
1453+
controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
1454+
initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
1455+
1456+
addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
1457+
1458+
workerMachine := newWorkerMachineForCluster(cluster)
1459+
workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
1460+
addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
1461+
objects := []client.Object{
1462+
cluster,
1463+
workerMachine,
1464+
workerJoinConfig,
1465+
}
1466+
1467+
objects = append(objects, createSecrets(t, cluster, initConfig)...)
1468+
myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
1469+
remoteClient := fake.NewClientBuilder().Build()
1470+
k := &KubeadmConfigReconciler{
1471+
Client: myclient,
1472+
SecretCachingClient: myclient,
1473+
KubeadmInitLock: &myInitLocker{},
1474+
TokenTTL: DefaultTokenTTL,
1475+
ClusterCache: clustercache.NewFakeClusterCache(remoteClient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
1476+
}
1477+
request := ctrl.Request{
1478+
NamespacedName: client.ObjectKey{
1479+
Namespace: metav1.NamespaceDefault,
1480+
Name: "worker-join-cfg",
1481+
},
1482+
}
1483+
result, err := k.Reconcile(ctx, request)
1484+
g.Expect(err).ToNot(HaveOccurred())
1485+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1486+
1487+
cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
1488+
g.Expect(err).ToNot(HaveOccurred())
1489+
g.Expect(cfg.Status.Ready).To(BeTrue())
1490+
g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
1491+
g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
1492+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1493+
firstToken := cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
1494+
1495+
l := &corev1.SecretList{}
1496+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1497+
g.Expect(l.Items).To(HaveLen(1))
1498+
1499+
t.Log("Token should not get recreated for single Machine since it will not use the new token if spec.bootstrap.dataSecretName was already set")
1500+
1501+
// Simulate token cleaner of Kubernetes having deleted the token secret
1502+
err = remoteClient.Delete(ctx, &l.Items[0])
1503+
g.Expect(err).ToNot(HaveOccurred())
1504+
1505+
result, err = k.Reconcile(ctx, request)
1506+
g.Expect(err).To(HaveOccurred())
1507+
g.Expect(err.Error()).To(ContainSubstring("failed to get bootstrap token secret in order to refresh it"))
1508+
// New token should not have been created
1509+
cfg, err = getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
1510+
g.Expect(err).ToNot(HaveOccurred())
1511+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).To(Equal(firstToken))
1512+
1513+
l = &corev1.SecretList{}
1514+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1515+
g.Expect(l.Items).To(BeEmpty())
1516+
})
1517+
t.Run("should recreate the token for MachinePools", func(t *testing.T) {
1518+
_ = feature.MutableGates.Set("MachinePool=true")
1519+
g := NewWithT(t)
1520+
1521+
cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
1522+
cluster.Status.InfrastructureReady = true
1523+
conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
1524+
cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
1525+
1526+
controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
1527+
initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
1528+
1529+
addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
1530+
1531+
workerMachinePool := newWorkerMachinePoolForCluster(cluster)
1532+
workerJoinConfig := newWorkerJoinKubeadmConfig(workerMachinePool.Namespace, "workerpool-join-cfg")
1533+
addKubeadmConfigToMachinePool(workerJoinConfig, workerMachinePool)
1534+
objects := []client.Object{
1535+
cluster,
1536+
workerMachinePool,
1537+
workerJoinConfig,
1538+
}
1539+
1540+
objects = append(objects, createSecrets(t, cluster, initConfig)...)
1541+
myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &expv1.MachinePool{}).Build()
1542+
remoteClient := fake.NewClientBuilder().Build()
1543+
k := &KubeadmConfigReconciler{
1544+
Client: myclient,
1545+
SecretCachingClient: myclient,
1546+
KubeadmInitLock: &myInitLocker{},
1547+
TokenTTL: DefaultTokenTTL,
1548+
ClusterCache: clustercache.NewFakeClusterCache(remoteClient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
1549+
}
1550+
request := ctrl.Request{
1551+
NamespacedName: client.ObjectKey{
1552+
Namespace: metav1.NamespaceDefault,
1553+
Name: "workerpool-join-cfg",
1554+
},
1555+
}
1556+
result, err := k.Reconcile(ctx, request)
1557+
g.Expect(err).ToNot(HaveOccurred())
1558+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1559+
1560+
cfg, err := getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
1561+
g.Expect(err).ToNot(HaveOccurred())
1562+
g.Expect(cfg.Status.Ready).To(BeTrue())
1563+
g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
1564+
g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
1565+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1566+
firstToken := cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
1567+
1568+
l := &corev1.SecretList{}
1569+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1570+
g.Expect(l.Items).To(HaveLen(1))
1571+
1572+
t.Log("Ensure that the token gets recreated if it was cleaned up by Kubernetes (e.g. on expiry)")
1573+
1574+
// Simulate token cleaner of Kubernetes having deleted the token secret
1575+
err = remoteClient.Delete(ctx, &l.Items[0])
1576+
g.Expect(err).ToNot(HaveOccurred())
1577+
1578+
result, err = k.Reconcile(ctx, request)
1579+
g.Expect(err).ToNot(HaveOccurred())
1580+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1581+
// New token should have been created
1582+
cfg, err = getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
1583+
g.Expect(err).ToNot(HaveOccurred())
1584+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1585+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(Equal(firstToken))
1586+
1587+
l = &corev1.SecretList{}
1588+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1589+
g.Expect(l.Items).To(HaveLen(1))
1590+
})
1591+
}
1592+
14441593
// Ensure the discovery portion of the JoinConfiguration gets generated correctly.
14451594
func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileBehaviors(t *testing.T) {
14461595
caHash := []string{"...."}

0 commit comments

Comments
 (0)