Skip to content

Commit 2b60ccd

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4575 from CecileRobertMichon/machinepool-logs
⚠️ Add MachinePool to e2e framework log collector
2 parents 04c2635 + 5f50069 commit 2b60ccd

File tree

4 files changed

+86
-46
lines changed

4 files changed

+86
-46
lines changed

‎test/e2e/common.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterPr
6363
Byf("Dumping logs from the %q workload cluster", cluster.Name)
6464

6565
// Dump all the logs from the workload cluster before deleting them.
66-
clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name, "machines"))
66+
clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name))
6767

6868
Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name)
6969

‎test/framework/cluster_proxy.go

+31-3
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@ package framework
1818

1919
import (
2020
"context"
21+
"errors"
2122
"fmt"
2223
"net/url"
2324
"os"
2425
"path"
2526
goruntime "runtime"
27+
expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
2628
"strings"
2729

2830
. "github.com/onsi/gomega"
@@ -81,6 +83,7 @@ type ClusterLogCollector interface {
8183
// CollectMachineLog collects log from a machine.
8284
// TODO: describe output folder struct
8385
CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error
86+
CollectMachinePoolLog(ctx context.Context, managementClusterClient client.Client, m *expv1.MachinePool, outputPath string) error
8487
}
8588

8689
// Option is a configuration option supplied to NewClusterProxy.
@@ -226,29 +229,54 @@ func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace
226229

227230
for i := range machines.Items {
228231
m := &machines.Items[i]
229-
err := p.logCollector.CollectMachineLog(ctx, p.GetClient(), m, path.Join(outputPath, m.GetName()))
232+
err := p.logCollector.CollectMachineLog(ctx, p.GetClient(), m, path.Join(outputPath, "machines", m.GetName()))
230233
if err != nil {
231234
// NB. we are treating failures in collecting logs as a non blocking operation (best effort)
232235
fmt.Printf("Failed to get logs for machine %s, cluster %s/%s: %v\n", m.GetName(), namespace, name, err)
233236
}
234237
}
238+
239+
machinePools, err := getMachinePoolsInCluster(ctx, p.GetClient(), namespace, name)
240+
Expect(err).ToNot(HaveOccurred(), "Failed to get machine pools for the %s/%s cluster", namespace, name)
241+
242+
for i := range machinePools.Items {
243+
mp := &machinePools.Items[i]
244+
err := p.logCollector.CollectMachinePoolLog(ctx, p.GetClient(), mp, path.Join(outputPath, "machine-pools", mp.GetName()))
245+
if err != nil {
246+
// NB. we are treating failures in collecting logs as a non blocking operation (best effort)
247+
fmt.Printf("Failed to get logs for machine pool %s, cluster %s/%s: %v\n", mp.GetName(), namespace, name, err)
248+
}
249+
}
235250
}
236251

237252
func getMachinesInCluster(ctx context.Context, c client.Client, namespace, name string) (*clusterv1.MachineList, error) {
238253
if name == "" {
239-
return nil, nil
254+
return nil, errors.New("cluster name should not be empty")
240255
}
241256

242257
machineList := &clusterv1.MachineList{}
243258
labels := map[string]string{clusterv1.ClusterLabelName: name}
244-
245259
if err := c.List(ctx, machineList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil {
246260
return nil, err
247261
}
248262

249263
return machineList, nil
250264
}
251265

266+
func getMachinePoolsInCluster(ctx context.Context, c client.Client, namespace, name string) (*expv1.MachinePoolList, error) {
267+
if name == "" {
268+
return nil, errors.New("cluster name should not be empty")
269+
}
270+
271+
machinePoolList := &expv1.MachinePoolList{}
272+
labels := map[string]string{clusterv1.ClusterLabelName: name}
273+
if err := c.List(ctx, machinePoolList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil {
274+
return nil, err
275+
}
276+
277+
return machinePoolList, nil
278+
}
279+
252280
func (p *clusterProxy) getKubeconfig(ctx context.Context, namespace string, name string) *api.Config {
253281
cl := p.GetClient()
254282

‎test/framework/docker_logcollector.go

+18
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ package framework
1919
import (
2020
"context"
2121
"fmt"
22+
kerrors "k8s.io/apimachinery/pkg/util/errors"
2223
"os"
2324
osExec "os/exec"
2425
"path/filepath"
26+
expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
2527
"strings"
2628

2729
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
@@ -45,6 +47,22 @@ func machineContainerName(cluster, machine string) string {
4547

4648
func (k DockerLogCollector) CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error {
4749
containerName := machineContainerName(m.Spec.ClusterName, m.Name)
50+
return k.collectLogsFromNode(outputPath, containerName)
51+
}
52+
53+
func (k DockerLogCollector) CollectMachinePoolLog(ctx context.Context, managementClusterClient client.Client, m *expv1.MachinePool, outputPath string) error {
54+
var errs []error
55+
for _, instance := range m.Status.NodeRefs {
56+
containerName := machineContainerName(m.Spec.ClusterName, instance.Name)
57+
if err := k.collectLogsFromNode(filepath.Join(outputPath, instance.Name), containerName); err != nil {
58+
// collecting logs is best effort so we proceed to the next instance even if we encounter an error.
59+
errs = append(errs, err)
60+
}
61+
}
62+
return kerrors.NewAggregate(errs)
63+
}
64+
65+
func (k DockerLogCollector) collectLogsFromNode(outputPath string, containerName string) error {
4866
execToPathFn := func(outputFileName, command string, args ...string) func() error {
4967
return func() error {
5068
f, err := fileOnHost(filepath.Join(outputPath, outputFileName))

‎test/framework/machinepool_helpers.go

+36-42
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"context"
2121
"github.com/pkg/errors"
2222
corev1 "k8s.io/api/core/v1"
23-
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2423
"sigs.k8s.io/cluster-api/test/framework/internal/log"
2524
"sigs.k8s.io/cluster-api/util/patch"
2625

@@ -92,7 +91,7 @@ type DiscoveryAndWaitForMachinePoolsInput struct {
9291
Cluster *clusterv1.Cluster
9392
}
9493

95-
// DiscoveryAndWaitForMachinePools discovers the MachinePools existing in a cluster and waits for them to be ready (all the machine provisioned).
94+
// DiscoveryAndWaitForMachinePools discovers the MachinePools existing in a cluster and waits for them to be ready (all the machines provisioned).
9695
func DiscoveryAndWaitForMachinePools(ctx context.Context, input DiscoveryAndWaitForMachinePoolsInput, intervals ...interface{}) []*clusterv1exp.MachinePool {
9796
Expect(ctx).NotTo(BeNil(), "ctx is required for DiscoveryAndWaitForMachinePools")
9897
Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling DiscoveryAndWaitForMachinePools")
@@ -131,25 +130,23 @@ func UpgradeMachinePoolAndWait(ctx context.Context, input UpgradeMachinePoolAndW
131130
mgmtClient := input.ClusterProxy.GetClient()
132131
for i := range input.MachinePools {
133132
mp := input.MachinePools[i]
134-
log.Logf("Patching the new kubernetes version to Machine Pool %s/%s", mp.Namespace, mp.Name)
133+
log.Logf("Patching the new Kubernetes version to Machine Pool %s/%s", mp.Namespace, mp.Name)
135134
patchHelper, err := patch.NewHelper(mp, mgmtClient)
136135
Expect(err).ToNot(HaveOccurred())
137136

137+
oldVersion := mp.Spec.Template.Spec.Version
138138
mp.Spec.Template.Spec.Version = &input.UpgradeVersion
139139
Expect(patchHelper.Patch(ctx, mp)).To(Succeed())
140-
}
141140

142-
for i := range input.MachinePools {
143-
mp := input.MachinePools[i]
144-
oldVersion := mp.Spec.Template.Spec.Version
145141
log.Logf("Waiting for Kubernetes versions of machines in MachinePool %s/%s to be upgraded from %s to %s",
146142
mp.Namespace, mp.Name, *oldVersion, input.UpgradeVersion)
147143
WaitForMachinePoolInstancesToBeUpgraded(ctx, WaitForMachinePoolInstancesToBeUpgradedInput{
148144
Getter: mgmtClient,
145+
WorkloadClusterGetter: input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient(),
149146
Cluster: input.Cluster,
150147
MachineCount: int(*mp.Spec.Replicas),
151148
KubernetesUpgradeVersion: input.UpgradeVersion,
152-
MachinePool: *mp,
149+
MachinePool: mp,
153150
}, input.WaitForMachinePoolToBeUpgraded...)
154151
}
155152
}
@@ -190,10 +187,11 @@ func ScaleMachinePoolAndWait(ctx context.Context, input ScaleMachinePoolAndWaitI
190187
// WaitForMachinePoolInstancesToBeUpgradedInput is the input for WaitForMachinePoolInstancesToBeUpgraded.
191188
type WaitForMachinePoolInstancesToBeUpgradedInput struct {
192189
Getter Getter
190+
WorkloadClusterGetter Getter
193191
Cluster *clusterv1.Cluster
194192
KubernetesUpgradeVersion string
195193
MachineCount int
196-
MachinePool clusterv1exp.MachinePool
194+
MachinePool *clusterv1exp.MachinePool
197195
}
198196

199197
// WaitForMachinePoolInstancesToBeUpgraded waits until all instances belonging to a MachinePool are upgraded to the correct kubernetes version.
@@ -207,10 +205,17 @@ func WaitForMachinePoolInstancesToBeUpgraded(ctx context.Context, input WaitForM
207205

208206
log.Logf("Ensuring all MachinePool Instances have upgraded kubernetes version %s", input.KubernetesUpgradeVersion)
209207
Eventually(func() (int, error) {
210-
versions := GetMachinePoolInstanceVersions(ctx, GetMachinesPoolInstancesInput{
211-
Getter: input.Getter,
212-
Namespace: input.Cluster.Namespace,
213-
MachinePool: input.MachinePool,
208+
nn := client.ObjectKey{
209+
Namespace: input.MachinePool.Namespace,
210+
Name: input.MachinePool.Name,
211+
}
212+
if err := input.Getter.Get(ctx, nn, input.MachinePool); err != nil {
213+
return 0, err
214+
}
215+
versions := getMachinePoolInstanceVersions(ctx, GetMachinesPoolInstancesInput{
216+
WorkloadClusterGetter: input.WorkloadClusterGetter,
217+
Namespace: input.Cluster.Namespace,
218+
MachinePool: input.MachinePool,
214219
})
215220

216221
matches := 0
@@ -230,41 +235,30 @@ func WaitForMachinePoolInstancesToBeUpgraded(ctx context.Context, input WaitForM
230235

231236
// GetMachinesPoolInstancesInput is the input for GetMachinesPoolInstances.
232237
type GetMachinesPoolInstancesInput struct {
233-
Getter Getter
234-
Namespace string
235-
MachinePool clusterv1exp.MachinePool
238+
WorkloadClusterGetter Getter
239+
Namespace string
240+
MachinePool *clusterv1exp.MachinePool
236241
}
237242

238-
// GetMachinePoolInstanceVersions returns the.
239-
func GetMachinePoolInstanceVersions(ctx context.Context, input GetMachinesPoolInstancesInput) []string {
240-
Expect(ctx).NotTo(BeNil(), "ctx is required for GetMachinePoolInstanceVersions")
241-
Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling GetMachinePoolInstanceVersions")
242-
Expect(input.MachinePool).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling GetMachinePoolInstanceVersions")
243-
244-
obj := getUnstructuredRef(ctx, input.Getter, &input.MachinePool.Spec.Template.Spec.InfrastructureRef, input.Namespace)
245-
instances, found, err := unstructured.NestedSlice(obj.Object, "status", "instances")
246-
Expect(err).ToNot(HaveOccurred(), "failed to extract machines from unstructured")
247-
if !found {
248-
return nil
249-
}
243+
// getMachinePoolInstanceVersions returns the Kubernetes versions of the machine pool instances.
244+
func getMachinePoolInstanceVersions(ctx context.Context, input GetMachinesPoolInstancesInput) []string {
245+
Expect(ctx).NotTo(BeNil(), "ctx is required for getMachinePoolInstanceVersions")
246+
Expect(input.WorkloadClusterGetter).ToNot(BeNil(), "Invalid argument. input.WorkloadClusterGetter can't be nil when calling getMachinePoolInstanceVersions")
247+
Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling getMachinePoolInstanceVersions")
248+
Expect(input.MachinePool).ToNot(BeNil(), "Invalid argument. input.MachinePool can't be nil when calling getMachinePoolInstanceVersions")
250249

250+
instances := input.MachinePool.Status.NodeRefs
251251
versions := make([]string, len(instances))
252252
for i, instance := range instances {
253-
version, found, err := unstructured.NestedString(instance.(map[string]interface{}), "version")
254-
Expect(err).ToNot(HaveOccurred(), "failed to extract versions from unstructured instance")
255-
Expect(found).To(BeTrue(), "unable to find nested version string in unstructured instance")
256-
versions[i] = version
253+
node := &corev1.Node{}
254+
err := input.WorkloadClusterGetter.Get(ctx, client.ObjectKey{Name: instance.Name}, node)
255+
if err != nil {
256+
versions[i] = "unknown"
257+
} else {
258+
versions[i] = node.Status.NodeInfo.KubeletVersion
259+
}
260+
log.Logf("Node %s version is %s", instance.Name, versions[i])
257261
}
258262

259263
return versions
260264
}
261-
262-
func getUnstructuredRef(ctx context.Context, getter Getter, ref *corev1.ObjectReference, namespace string) *unstructured.Unstructured {
263-
obj := new(unstructured.Unstructured)
264-
obj.SetAPIVersion(ref.APIVersion)
265-
obj.SetKind(ref.Kind)
266-
obj.SetName(ref.Name)
267-
key := client.ObjectKey{Name: obj.GetName(), Namespace: namespace}
268-
Expect(getter.Get(ctx, key, obj)).ToNot(HaveOccurred(), "failed to retrieve %s object %q/%q", obj.GetKind(), key.Namespace, key.Name)
269-
return obj
270-
}

0 commit comments

Comments
 (0)