Skip to content

Commit cb6ab45

Browse files
yevgeny-shnaidmanyevgeny-shnaidman
authored
Loading multiple independent kernel modules in one KMM Module (#381)
This PR allows defining the load order of multiple independent kernel modules in one KMM Module. The order is defined as a list in the ModulesLoadingOrder field of the ModprobeSpec.The first module will be loaded last, the last one in the list will be loaded first. It is implement by using DownwardAPI: module loader pod template contains is added an annotation that contains the /etc/modprobe.d/softdep format, and then this annotation is added as a file to module loader's pod filesystem using Downward API
1 parent 2b20981 commit cb6ab45

File tree

2 files changed

+113
-30
lines changed

2 files changed

+113
-30
lines changed

‎internal/daemonset/daemonset.go

+56-18
Original file line numberDiff line numberDiff line change
@@ -57,19 +57,12 @@ func (dc *daemonSetGenerator) GarbageCollect(ctx context.Context, mod *kmmv1beta
5757
deleted := make([]string, 0)
5858

5959
for _, ds := range existingDS {
60-
if isOlderVersionUnusedDaemonset(&ds, mod.Spec.ModuleLoader.Container.Version) {
61-
if err := dc.client.Delete(ctx, &ds); err != nil {
62-
return nil, fmt.Errorf("could not delete DaemonSet %s: %v", ds.Name, err)
63-
}
60+
if isOlderVersionUnusedDaemonset(&ds, mod.Spec.ModuleLoader.Container.Version) ||
61+
isModuleLoaderDaemonsetWithInvalidKernel(&ds, dc.kernelLabel, validKernels) {
6462
deleted = append(deleted, ds.Name)
65-
continue
66-
}
67-
if isModuleLoaderDaemonsetWithInvalidKernel(&ds, dc.kernelLabel, validKernels) {
6863
if err := dc.client.Delete(ctx, &ds); err != nil {
6964
return nil, fmt.Errorf("could not delete DaemonSet %s: %v", ds.Name, err)
7065
}
71-
72-
deleted = append(deleted, ds.Name)
7366
}
7467
}
7568

@@ -125,12 +118,12 @@ func (dc *daemonSetGenerator) SetDriverContainerAsDesired(
125118
Lifecycle: &v1.Lifecycle{
126119
PostStart: &v1.LifecycleHandler{
127120
Exec: &v1.ExecAction{
128-
Command: MakeLoadCommand(mld.InTreeRemoval, mld.Modprobe, mld.Name),
121+
Command: makeLoadCommand(mld.InTreeRemoval, mld.Modprobe, mld.Name),
129122
},
130123
},
131124
PreStop: &v1.LifecycleHandler{
132125
Exec: &v1.ExecAction{
133-
Command: MakeUnloadCommand(mld.Modprobe, mld.Name),
126+
Command: makeUnloadCommand(mld.Modprobe, mld.Name),
134127
},
135128
},
136129
},
@@ -185,11 +178,40 @@ func (dc *daemonSetGenerator) SetDriverContainerAsDesired(
185178
container.VolumeMounts = append(container.VolumeMounts, firmwareVolumeMount)
186179
}
187180

181+
var modulesOrderAnnotations map[string]string
182+
if mld.Modprobe.ModulesLoadingOrder != nil {
183+
modulesOrderAnnotations = map[string]string{
184+
"modules-order": getModulesOrderAnnotationValue(mld),
185+
}
186+
softdepVolume := v1.Volume{
187+
Name: "modules-order",
188+
VolumeSource: v1.VolumeSource{
189+
DownwardAPI: &v1.DownwardAPIVolumeSource{
190+
Items: []v1.DownwardAPIVolumeFile{
191+
{
192+
Path: "softdep.conf",
193+
FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.annotations['modules-order']"},
194+
},
195+
},
196+
},
197+
},
198+
}
199+
volumes = append(volumes, softdepVolume)
200+
201+
softDepVolumeMount := v1.VolumeMount{
202+
Name: "modules-order",
203+
ReadOnly: true,
204+
MountPath: "/etc/modprobe.d",
205+
}
206+
container.VolumeMounts = append(container.VolumeMounts, softDepVolumeMount)
207+
}
208+
188209
ds.Spec = appsv1.DaemonSetSpec{
189210
Template: v1.PodTemplateSpec{
190211
ObjectMeta: metav1.ObjectMeta{
191-
Labels: standardLabels,
192-
Finalizers: []string{constants.NodeLabelerFinalizer},
212+
Labels: standardLabels,
213+
Finalizers: []string{constants.NodeLabelerFinalizer},
214+
Annotations: modulesOrderAnnotations,
193215
},
194216
Spec: v1.PodSpec{
195217
ShareProcessNamespace: swag.Bool(true),
@@ -276,9 +298,8 @@ func (dc *daemonSetGenerator) SetDevicePluginAsDesired(
276298
VolumeMounts: append(mod.Spec.DevicePlugin.Container.VolumeMounts, containerVolumeMounts...),
277299
},
278300
},
279-
PriorityClassName: "system-node-critical",
280-
ImagePullSecrets: GetPodPullSecrets(mod.Spec.ImageRepoSecret),
281-
//NodeSelector: map[string]string{getDriverContainerNodeLabel(mod.Namespace, mod.Name, true): ""},
301+
PriorityClassName: "system-node-critical",
302+
ImagePullSecrets: GetPodPullSecrets(mod.Spec.ImageRepoSecret),
282303
NodeSelector: nodeSelector,
283304
ServiceAccountName: mod.Spec.DevicePlugin.ServiceAccountName,
284305
Volumes: append([]v1.Volume{devicePluginVolume}, mod.Spec.DevicePlugin.Volumes...),
@@ -375,7 +396,7 @@ func OverrideLabels(labels, overrides map[string]string) map[string]string {
375396
return labels
376397
}
377398

378-
func MakeLoadCommand(inTreeRemoval bool, spec kmmv1beta1.ModprobeSpec, modName string) []string {
399+
func makeLoadCommand(inTreeRemoval bool, spec kmmv1beta1.ModprobeSpec, modName string) []string {
379400
loadCommandShell := []string{
380401
"/bin/sh",
381402
"-c",
@@ -430,7 +451,7 @@ func MakeLoadCommand(inTreeRemoval bool, spec kmmv1beta1.ModprobeSpec, modName s
430451
return append(loadCommandShell, loadCommand.String())
431452
}
432453

433-
func MakeUnloadCommand(spec kmmv1beta1.ModprobeSpec, modName string) []string {
454+
func makeUnloadCommand(spec kmmv1beta1.ModprobeSpec, modName string) []string {
434455
unloadCommandShell := []string{
435456
"/bin/sh",
436457
"-c",
@@ -472,3 +493,20 @@ func MakeUnloadCommand(spec kmmv1beta1.ModprobeSpec, modName string) []string {
472493

473494
return append(unloadCommandShell, unloadCommand.String())
474495
}
496+
497+
func getModulesOrderAnnotationValue(mld *api.ModuleLoaderData) string {
498+
modulesNames := mld.Modprobe.ModulesLoadingOrder
499+
softDepData := ""
500+
for i := 0; i < len(modulesNames); i++ {
501+
if i == len(modulesNames)-1 {
502+
break
503+
}
504+
line := prepareSoftDepLine(modulesNames[i], modulesNames[i+1])
505+
softDepData = softDepData + line
506+
}
507+
return softDepData
508+
}
509+
510+
func prepareSoftDepLine(dependendModuleName, moduleName string) string {
511+
return fmt.Sprintf("softdep %s pre: %s\n", dependendModuleName, moduleName)
512+
}

‎internal/daemonset/daemonset_test.go

+57-12
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,51 @@ var _ = Describe("SetDriverContainerAsDesired", func() {
8787
Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(1))
8888
})
8989

90+
It("should add volume and volume mount if module ordering is set", func() {
91+
mld := api.ModuleLoaderData{
92+
Name: moduleName,
93+
Namespace: namespace,
94+
Modprobe: kmmv1beta1.ModprobeSpec{
95+
ModulesLoadingOrder: []string{"ModuleA", "ModuleB", "ModuleC"},
96+
},
97+
Owner: &kmmv1beta1.Module{},
98+
ContainerImage: "some image",
99+
KernelVersion: kernelVersion,
100+
ModuleVersion: "some version",
101+
}
102+
103+
orderedVol := v1.Volume{
104+
Name: "modules-order",
105+
VolumeSource: v1.VolumeSource{
106+
DownwardAPI: &v1.DownwardAPIVolumeSource{
107+
Items: []v1.DownwardAPIVolumeFile{
108+
{
109+
Path: "softdep.conf",
110+
FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.annotations['modules-order']"},
111+
},
112+
},
113+
},
114+
},
115+
}
116+
orderedVolMount := v1.VolumeMount{
117+
Name: "modules-order",
118+
ReadOnly: true,
119+
MountPath: "/etc/modprobe.d",
120+
}
121+
122+
annotations := map[string]string{"modules-order": "softdep ModuleA pre: ModuleB\nsoftdep ModuleB pre: ModuleC\n"}
123+
124+
ds := appsv1.DaemonSet{}
125+
126+
err := dc.SetDriverContainerAsDesired(context.Background(), &ds, &mld)
127+
Expect(err).NotTo(HaveOccurred())
128+
Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(2))
129+
Expect(ds.Spec.Template.Spec.Volumes[1]).To(Equal(orderedVol))
130+
Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(2))
131+
Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts[1]).To(Equal(orderedVolMount))
132+
Expect(ds.Spec.Template.Annotations).To(Equal(annotations))
133+
})
134+
90135
It("should add the volume and volume mount for firmware if FirmwarePath is set", func() {
91136
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
92137
vol := v1.Volume{
@@ -224,12 +269,12 @@ var _ = Describe("SetDriverContainerAsDesired", func() {
224269
Lifecycle: &v1.Lifecycle{
225270
PostStart: &v1.LifecycleHandler{
226271
Exec: &v1.ExecAction{
227-
Command: MakeLoadCommand(mld.InTreeRemoval, mld.Modprobe, moduleName),
272+
Command: makeLoadCommand(mld.InTreeRemoval, mld.Modprobe, moduleName),
228273
},
229274
},
230275
PreStop: &v1.LifecycleHandler{
231276
Exec: &v1.ExecAction{
232-
Command: MakeUnloadCommand(mld.Modprobe, moduleName),
277+
Command: makeUnloadCommand(mld.Modprobe, moduleName),
233278
},
234279
},
235280
},
@@ -803,7 +848,7 @@ var _ = Describe("GetNodeLabelFromPod", func() {
803848
})
804849
})
805850

806-
var _ = Describe("MakeLoadCommand", func() {
851+
var _ = Describe("makeLoadCommand", func() {
807852
const (
808853
kernelModuleName = "some-kmod"
809854
moduleName = "module-name"
@@ -818,7 +863,7 @@ var _ = Describe("MakeLoadCommand", func() {
818863
}
819864

820865
Expect(
821-
MakeLoadCommand(false, spec, moduleName),
866+
makeLoadCommand(false, spec, moduleName),
822867
).To(
823868
Equal([]string{
824869
"/bin/sh",
@@ -842,7 +887,7 @@ var _ = Describe("MakeLoadCommand", func() {
842887
}
843888

844889
Expect(
845-
MakeLoadCommand(true, spec, moduleName),
890+
makeLoadCommand(true, spec, moduleName),
846891
).To(
847892
Equal([]string{
848893
"/bin/sh",
@@ -861,7 +906,7 @@ var _ = Describe("MakeLoadCommand", func() {
861906
}
862907

863908
Expect(
864-
MakeLoadCommand(false, spec, moduleName),
909+
makeLoadCommand(false, spec, moduleName),
865910
).To(
866911
Equal([]string{
867912
"/bin/sh",
@@ -878,7 +923,7 @@ var _ = Describe("MakeLoadCommand", func() {
878923
}
879924

880925
Expect(
881-
MakeLoadCommand(false, spec, moduleName),
926+
makeLoadCommand(false, spec, moduleName),
882927
).To(
883928
Equal([]string{
884929
"/bin/sh",
@@ -889,7 +934,7 @@ var _ = Describe("MakeLoadCommand", func() {
889934
})
890935
})
891936

892-
var _ = Describe("MakeUnloadCommand", func() {
937+
var _ = Describe("makeUnloadCommand", func() {
893938
const (
894939
kernelModuleName = "some-kmod"
895940
moduleName = "module-name"
@@ -904,7 +949,7 @@ var _ = Describe("MakeUnloadCommand", func() {
904949
}
905950

906951
Expect(
907-
MakeUnloadCommand(spec, moduleName),
952+
makeUnloadCommand(spec, moduleName),
908953
).To(
909954
Equal([]string{
910955
"/bin/sh",
@@ -923,7 +968,7 @@ var _ = Describe("MakeUnloadCommand", func() {
923968
}
924969

925970
Expect(
926-
MakeUnloadCommand(spec, moduleName),
971+
makeUnloadCommand(spec, moduleName),
927972
).To(
928973
Equal([]string{
929974
"/bin/sh",
@@ -942,7 +987,7 @@ var _ = Describe("MakeUnloadCommand", func() {
942987
}
943988

944989
Expect(
945-
MakeUnloadCommand(spec, moduleName),
990+
makeUnloadCommand(spec, moduleName),
946991
).To(
947992
Equal([]string{
948993
"/bin/sh",
@@ -959,7 +1004,7 @@ var _ = Describe("MakeUnloadCommand", func() {
9591004
}
9601005

9611006
Expect(
962-
MakeUnloadCommand(spec, moduleName),
1007+
makeUnloadCommand(spec, moduleName),
9631008
).To(
9641009
Equal([]string{
9651010
"/bin/sh",

0 commit comments

Comments
 (0)