Skip to content

Commit 1963141

Browse files
committed
kvm2 driver support simulate numa node
1 parent a99c6c2 commit 1963141

File tree

8 files changed

+168
-4
lines changed

8 files changed

+168
-4
lines changed

cmd/minikube/cmd/start_flags.go

+7
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ const (
6565
kvmQemuURI = "kvm-qemu-uri"
6666
kvmGPU = "kvm-gpu"
6767
kvmHidden = "kvm-hidden"
68+
kvmNUMACount = "kvm-numa-count"
6869
minikubeEnvPrefix = "MINIKUBE"
6970
installAddons = "install-addons"
7071
defaultDiskSize = "20000mb"
@@ -193,6 +194,7 @@ func initDriverFlags() {
193194
startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)")
194195
startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube")
195196
startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)")
197+
startCmd.Flags().Int(kvmNUMACount, 1, "Simulate numa node count in minikube. (kvm2 driver only)")
196198

197199
// virtualbox
198200
startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)")
@@ -338,6 +340,7 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
338340
KVMQemuURI: viper.GetString(kvmQemuURI),
339341
KVMGPU: viper.GetBool(kvmGPU),
340342
KVMHidden: viper.GetBool(kvmHidden),
343+
KVMNUMACount: viper.GetInt(kvmNUMACount),
341344
DisableDriverMounts: viper.GetBool(disableDriverMounts),
342345
UUID: viper.GetString(uuid),
343346
NoVTXCheck: viper.GetBool(noVTXCheck),
@@ -545,6 +548,10 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
545548
cc.KVMHidden = viper.GetBool(kvmHidden)
546549
}
547550

551+
if cmd.Flags().Changed(kvmNUMACount) {
552+
cc.KVMNUMACount = viper.GetInt(kvmNUMACount)
553+
}
554+
548555
if cmd.Flags().Changed(disableDriverMounts) {
549556
cc.DisableDriverMounts = viper.GetBool(disableDriverMounts)
550557
}

pkg/drivers/kvm/domain.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,11 @@ const domainTmpl = `
4444
</kvm>
4545
{{end}}
4646
</features>
47-
<cpu mode='host-passthrough'/>
47+
<cpu mode='host-passthrough'>
48+
{{if gt .NUMANodeCount 1}}
49+
{{.NUMANodeXML}}
50+
{{end}}
51+
</cpu>
4852
<os>
4953
<type>hvm</type>
5054
<boot dev='cdrom'/>
@@ -158,14 +162,12 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) {
158162
}
159163
d.PrivateMAC = mac.String()
160164
}
161-
162165
// create the XML for the domain using our domainTmpl template
163166
tmpl := template.Must(template.New("domain").Parse(domainTmpl))
164167
var domainXML bytes.Buffer
165168
if err := tmpl.Execute(&domainXML, d); err != nil {
166169
return nil, errors.Wrap(err, "executing domain xml")
167170
}
168-
169171
conn, err := getConnection(d.ConnectionURI)
170172
if err != nil {
171173
return nil, errors.Wrap(err, "error getting libvirt connection")

pkg/drivers/kvm/kvm.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ type Driver struct {
8181

8282
// QEMU Connection URI
8383
ConnectionURI string
84+
85+
// NUMA node count default value is 1
86+
NUMANodeCount int
87+
88+
// NUMA XML
89+
NUMANodeXML string
8490
}
8591

8692
const (
@@ -301,7 +307,6 @@ func (d *Driver) Start() (err error) {
301307
func (d *Driver) Create() (err error) {
302308
log.Info("Creating KVM machine...")
303309
defer log.Infof("KVM machine creation complete!")
304-
305310
err = d.createNetwork()
306311
if err != nil {
307312
return errors.Wrap(err, "creating network")
@@ -314,6 +319,14 @@ func (d *Driver) Create() (err error) {
314319
}
315320
}
316321

322+
if d.NUMANodeCount > 1 {
323+
NUMAXML, err := GetNUMAXml(d.CPU, d.Memory, d.NUMANodeCount)
324+
if err != nil {
325+
return errors.Wrap(err, "creating NUMA XML")
326+
}
327+
d.NUMANodeXML = NUMAXML
328+
}
329+
317330
store := d.ResolveStorePath(".")
318331
log.Infof("Setting up store path in %s ...", store)
319332
// 0755 because it must be accessible by libvirt/qemu across a variety of configs

pkg/drivers/kvm/numa.go

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kvm
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
"strconv"
23+
"strings"
24+
"text/template"
25+
)
26+
27+
// NUMATmpl NUMA XML Template
28+
const NUMATmpl = `
29+
<numa>
30+
{{- range $idx,$val :=. }}
31+
<cell id='{{$idx}}' cpus='{{$val.CPUTopology}}' memory='{{$val.Memory}}' unit='MiB'/>
32+
{{- end }}
33+
</numa>
34+
`
35+
36+
// NUMA this struct use for NUMATmpl
37+
type NUMA struct {
38+
CPUCount int
39+
Memory int
40+
CPUTopology string
41+
}
42+
43+
// GetNUMAXml generate numa xml
44+
// evenly distributed cpu core & memory to each numa node
45+
func GetNUMAXml(cpu, memory, numaCount int) (string, error) {
46+
if numaCount < 1 {
47+
return "", fmt.Errorf("numa node count must >= 1")
48+
}
49+
if cpu < numaCount {
50+
return "", fmt.Errorf("cpu count must >= numa node count")
51+
}
52+
numaNodes := make([]*NUMA, numaCount)
53+
CPUSeq := 0
54+
cpuBaseCount := cpu / numaCount
55+
cpuExtraCount := cpu % numaCount
56+
57+
for i := range numaNodes {
58+
numaNodes[i] = &NUMA{CPUCount: cpuBaseCount}
59+
}
60+
61+
for i := 0; i < cpuExtraCount; i++ {
62+
numaNodes[i].CPUCount++
63+
}
64+
for i := range numaNodes {
65+
CPUTopologySlice := make([]string, 0)
66+
for seq := CPUSeq; seq < CPUSeq+numaNodes[i].CPUCount; seq++ {
67+
CPUTopologySlice = append(CPUTopologySlice, strconv.Itoa(seq))
68+
}
69+
numaNodes[i].CPUTopology = strings.Join(CPUTopologySlice, ",")
70+
CPUSeq += numaNodes[i].CPUCount
71+
}
72+
73+
memoryBaseCount := memory / numaCount
74+
memoryExtraCount := memory % numaCount
75+
76+
for i := range numaNodes {
77+
numaNodes[i].Memory = memoryBaseCount
78+
}
79+
80+
for i := 0; i < memoryExtraCount; i++ {
81+
numaNodes[i].Memory++
82+
}
83+
84+
tmpl := template.Must(template.New("numa").Parse(NUMATmpl))
85+
var NUMAXML bytes.Buffer
86+
if err := tmpl.Execute(&NUMAXML, numaNodes); err != nil {
87+
return "", fmt.Errorf("couldn't generate numa XML: %v", err)
88+
}
89+
return NUMAXML.String(), nil
90+
}

pkg/drivers/kvm/numa_test.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kvm
18+
19+
import (
20+
"strings"
21+
"testing"
22+
)
23+
24+
func TestGetNUMAXml(t *testing.T) {
25+
_, err := GetNUMAXml(1, 1024, 0)
26+
if err == nil {
27+
t.Errorf("check invalid numa count failed: %s", err)
28+
}
29+
30+
xml, err := GetNUMAXml(10, 10240, 8)
31+
expXML := `<numa>
32+
<cell id='0' cpus='0,1' memory='1280' unit='MiB'/>
33+
<cell id='1' cpus='2,3' memory='1280' unit='MiB'/>
34+
<cell id='2' cpus='4' memory='1280' unit='MiB'/>
35+
<cell id='3' cpus='5' memory='1280' unit='MiB'/>
36+
<cell id='4' cpus='6' memory='1280' unit='MiB'/>
37+
<cell id='5' cpus='7' memory='1280' unit='MiB'/>
38+
<cell id='6' cpus='8' memory='1280' unit='MiB'/>
39+
<cell id='7' cpus='9' memory='1280' unit='MiB'/>
40+
</numa>`
41+
if err != nil {
42+
t.Errorf("gen xml failed: %s", err)
43+
}
44+
if strings.TrimSpace(xml) != expXML {
45+
t.Errorf("gen xml: %s not match expect xml: %s", xml, expXML)
46+
}
47+
48+
}

pkg/minikube/config/types.go

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ type ClusterConfig struct {
5656
KVMQemuURI string // Only used by kvm2
5757
KVMGPU bool // Only used by kvm2
5858
KVMHidden bool // Only used by kvm2
59+
KVMNUMACount int // Only used by kvm2
5960
DockerOpt []string // Each entry is formatted as KEY=VALUE.
6061
DisableDriverMounts bool // Only used by virtualbox
6162
NFSShare []string

pkg/minikube/registry/drvs/kvm2/kvm2.go

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ type kvmDriver struct {
6767
GPU bool
6868
Hidden bool
6969
ConnectionURI string
70+
NUMANodeCount int
7071
}
7172

7273
func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
@@ -88,6 +89,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
8889
GPU: cc.KVMGPU,
8990
Hidden: cc.KVMHidden,
9091
ConnectionURI: cc.KVMQemuURI,
92+
NUMANodeCount: cc.KVMNUMACount,
9193
}, nil
9294
}
9395

site/content/en/docs/commands/start.md

+1
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ minikube start [flags]
7070
--kvm-gpu Enable experimental NVIDIA GPU support in minikube
7171
--kvm-hidden Hide the hypervisor signature from the guest in minikube (kvm2 driver only)
7272
--kvm-network string The KVM network name. (kvm2 driver only) (default "default")
73+
--kvm-numa-count int Simulate numa node count in minikube. (kvm2 driver only) (default 1)
7374
--kvm-qemu-uri string The KVM QEMU connection URI. (kvm2 driver only) (default "qemu:///system")
7475
--memory string Amount of RAM to allocate to Kubernetes (format: <number>[<unit>], where unit = b, k, m or g).
7576
--mount This will start the mount daemon and automatically mount files into minikube.

0 commit comments

Comments
 (0)