Skip to content

Commit a139ccd

Browse files
r2d4r2d4
committed
Implement extra-config for kubeadm kubelet
1 parent 93323c6 commit a139ccd

File tree

6 files changed

+522
-60
lines changed

6 files changed

+522
-60
lines changed

Diff for: ‎Makefile

+4
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,10 @@ endif
150150
test-iso:
151151
go test -v $(REPOPATH)/test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"
152152

153+
.PHONY: test-pkg
154+
test-pkg/%:
155+
go test -v -test.timeout=30m $(REPOPATH)/$* --tags="$(MINIKUBE_BUILD_TAGS)"
156+
153157
.PHONY: integration
154158
integration: out/minikube
155159
go test -v -test.timeout=30m $(REPOPATH)/test/integration --tags="$(MINIKUBE_INTEGRATION_BUILD_TAGS)" $(TEST_ARGS)

Diff for: ‎pkg/minikube/bootstrapper/kubeadm/kubeadm.go

+42-60
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"bytes"
2121
"crypto"
2222
"fmt"
23-
"html/template"
2423
"os"
2524
"path/filepath"
2625
"strings"
@@ -44,48 +43,6 @@ type KubeadmBootstrapper struct {
4443
c bootstrapper.CommandRunner
4544
}
4645

47-
// TODO(r2d4): template this with bootstrapper.KubernetesConfig
48-
const kubeletSystemdConf = `
49-
[Service]
50-
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
51-
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
52-
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
53-
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
54-
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
55-
ExecStart=
56-
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS
57-
`
58-
59-
const kubeletService = `
60-
[Unit]
61-
Description=kubelet: The Kubernetes Node Agent
62-
Documentation=http://kubernetes.io/docs/
63-
64-
[Service]
65-
ExecStart=/usr/bin/kubelet
66-
Restart=always
67-
StartLimitInterval=0
68-
RestartSec=10
69-
70-
[Install]
71-
WantedBy=multi-user.target
72-
`
73-
74-
const kubeadmConfigTmpl = `
75-
apiVersion: kubeadm.k8s.io/v1alpha1
76-
kind: MasterConfiguration
77-
api:
78-
advertiseAddress: {{.AdvertiseAddress}}
79-
bindPort: {{.APIServerPort}}
80-
kubernetesVersion: {{.KubernetesVersion}}
81-
certificatesDir: {{.CertDir}}
82-
networking:
83-
serviceSubnet: {{.ServiceCIDR}}
84-
etcd:
85-
dataDir: {{.EtcdDataDir}}
86-
nodeName: {{.NodeName}}
87-
`
88-
8946
func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) {
9047
h, err := api.Load(config.GetMachineName())
9148
if err != nil {
@@ -147,10 +104,8 @@ func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) {
147104
func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error {
148105
// We use --skip-preflight-checks since we have our own custom addons
149106
// that we also stick in /etc/kubernetes/manifests
150-
kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"
151-
t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl))
152107
b := bytes.Buffer{}
153-
if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
108+
if err := kubeadmInitTemplate.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
154109
return err
155110
}
156111

@@ -197,22 +152,14 @@ func addAddons(files *[]assets.CopyableFile) error {
197152
}
198153

199154
func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error {
200-
restoreTmpl := `
201-
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
202-
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
203-
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
204-
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
205-
`
206-
t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl))
207-
208155
opts := struct {
209156
KubeadmConfigFile string
210157
}{
211158
KubeadmConfigFile: constants.KubeadmConfigFile,
212159
}
213160

214161
b := bytes.Buffer{}
215-
if err := t.Execute(&b, opts); err != nil {
162+
if err := kubeadmRestoreTemplate.Execute(&b, opts); err != nil {
216163
return err
217164
}
218165

@@ -231,19 +178,44 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
231178
return bootstrapper.SetupCerts(k.c, k8s)
232179
}
233180

181+
func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
182+
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
183+
if err != nil {
184+
return "", errors.Wrap(err, "parsing kubernetes version")
185+
}
186+
187+
extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
188+
if err != nil {
189+
return "", errors.Wrap(err, "generating extra configuration for kubelet")
190+
}
191+
192+
extraFlags := convertToFlags(extraOpts)
193+
b := bytes.Buffer{}
194+
if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
195+
return "", err
196+
}
197+
198+
return b.String(), nil
199+
}
200+
234201
func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error {
235202
if cfg.ShouldLoadCachedImages {
236203
// Make best effort to load any cached images
237204
go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir)
238205
}
239-
kubeadmCfg, err := k.generateConfig(cfg)
206+
kubeadmCfg, err := generateConfig(cfg)
240207
if err != nil {
241208
return errors.Wrap(err, "generating kubeadm cfg")
242209
}
243210

211+
kubeletCfg, err := NewKubeletConfig(cfg)
212+
if err != nil {
213+
return errors.Wrap(err, "generating kubelet config")
214+
}
215+
244216
files := []assets.CopyableFile{
245217
assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"),
246-
assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"),
218+
assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"),
247219
assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"),
248220
}
249221

@@ -290,9 +262,17 @@ sudo systemctl start kubelet
290262
return nil
291263
}
292264

293-
func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
294-
t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl))
265+
func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
266+
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
267+
if err != nil {
268+
return "", errors.Wrap(err, "parsing kubernetes version")
269+
}
295270

271+
// generates a map of component to extra args for apiserver, controller-manager, and scheduler
272+
extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
273+
if err != nil {
274+
return "", errors.Wrap(err, "generating extra component config for kubeadm")
275+
}
296276
opts := struct {
297277
CertDir string
298278
ServiceCIDR string
@@ -301,6 +281,7 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
301281
KubernetesVersion string
302282
EtcdDataDir string
303283
NodeName string
284+
ExtraArgs []ComponentExtraArgs
304285
}{
305286
CertDir: util.DefaultCertPath,
306287
ServiceCIDR: util.DefaultInsecureRegistry,
@@ -309,10 +290,11 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
309290
KubernetesVersion: k8s.KubernetesVersion,
310291
EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted
311292
NodeName: k8s.NodeName,
293+
ExtraArgs: extraComponentConfig,
312294
}
313295

314296
b := bytes.Buffer{}
315-
if err := t.Execute(&b, opts); err != nil {
297+
if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil {
316298
return "", err
317299
}
318300

Diff for: ‎pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
/*
2+
Copyright 2016 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kubeadm
18+
19+
import (
20+
"testing"
21+
22+
"k8s.io/minikube/pkg/minikube/bootstrapper"
23+
"k8s.io/minikube/pkg/util"
24+
)
25+
26+
func TestGenerateConfig(t *testing.T) {
27+
tests := []struct {
28+
description string
29+
cfg bootstrapper.KubernetesConfig
30+
expectedCfg string
31+
shouldErr bool
32+
}{
33+
{
34+
description: "no extra args",
35+
cfg: bootstrapper.KubernetesConfig{
36+
NodeIP: "192.168.1.100",
37+
KubernetesVersion: "v1.8.0",
38+
NodeName: "minikube",
39+
},
40+
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
41+
kind: MasterConfiguration
42+
api:
43+
advertiseAddress: 192.168.1.100
44+
bindPort: 8443
45+
kubernetesVersion: v1.8.0
46+
certificatesDir: /var/lib/localkube/certs/
47+
networking:
48+
serviceSubnet: 10.0.0.0/24
49+
etcd:
50+
dataDir: /data
51+
nodeName: minikube
52+
`,
53+
},
54+
{
55+
description: "extra args all components",
56+
cfg: bootstrapper.KubernetesConfig{
57+
NodeIP: "192.168.1.101",
58+
KubernetesVersion: "v1.8.0-alpha.0",
59+
NodeName: "extra-args-minikube",
60+
ExtraOptions: util.ExtraOptionSlice{
61+
util.ExtraOption{
62+
Component: Apiserver,
63+
Key: "fail-no-swap",
64+
Value: "true",
65+
},
66+
util.ExtraOption{
67+
Component: ControllerManager,
68+
Key: "kube-api-burst",
69+
Value: "32",
70+
},
71+
util.ExtraOption{
72+
Component: Scheduler,
73+
Key: "scheduler-name",
74+
Value: "mini-scheduler",
75+
},
76+
},
77+
},
78+
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
79+
kind: MasterConfiguration
80+
api:
81+
advertiseAddress: 192.168.1.101
82+
bindPort: 8443
83+
kubernetesVersion: v1.8.0-alpha.0
84+
certificatesDir: /var/lib/localkube/certs/
85+
networking:
86+
serviceSubnet: 10.0.0.0/24
87+
etcd:
88+
dataDir: /data
89+
nodeName: extra-args-minikube
90+
apiServerExtraArgs:
91+
fail-no-swap: true
92+
controllerManagerExtraArgs:
93+
kube-api-burst: 32
94+
schedulerExtraArgs:
95+
scheduler-name: mini-scheduler
96+
`,
97+
},
98+
{
99+
// Unknown components should fail silently
100+
description: "unknown component",
101+
cfg: bootstrapper.KubernetesConfig{
102+
NodeIP: "192.168.1.101",
103+
KubernetesVersion: "v1.8.0-alpha.0",
104+
NodeName: "extra-args-minikube",
105+
ExtraOptions: util.ExtraOptionSlice{
106+
util.ExtraOption{
107+
Component: "not-a-real-component",
108+
Key: "killswitch",
109+
Value: "true",
110+
},
111+
},
112+
},
113+
shouldErr: true,
114+
},
115+
}
116+
117+
for _, test := range tests {
118+
t.Run(test.description, func(t *testing.T) {
119+
actualCfg, err := generateConfig(test.cfg)
120+
if err != nil && !test.shouldErr {
121+
t.Errorf("got unexpected error generating config: %s", err)
122+
return
123+
}
124+
if err == nil && test.shouldErr {
125+
t.Errorf("expected error but got none, config: %s", actualCfg)
126+
return
127+
}
128+
if actualCfg != test.expectedCfg {
129+
t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg)
130+
return
131+
}
132+
})
133+
}
134+
}

Diff for: ‎pkg/minikube/bootstrapper/kubeadm/templates.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
Copyright 2016 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kubeadm
18+
19+
import "html/template"
20+
21+
var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
22+
kind: MasterConfiguration
23+
api:
24+
advertiseAddress: {{.AdvertiseAddress}}
25+
bindPort: {{.APIServerPort}}
26+
kubernetesVersion: {{.KubernetesVersion}}
27+
certificatesDir: {{.CertDir}}
28+
networking:
29+
serviceSubnet: {{.ServiceCIDR}}
30+
etcd:
31+
dataDir: {{.EtcdDataDir}}
32+
nodeName: {{.NodeName}}
33+
{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
34+
{{$key}}: {{$value}}
35+
{{end}}{{end}}`))
36+
37+
var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
38+
[Service]
39+
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
40+
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
41+
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
42+
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
43+
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
44+
ExecStart=
45+
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
46+
`))
47+
48+
const kubeletService = `
49+
[Unit]
50+
Description=kubelet: The Kubernetes Node Agent
51+
Documentation=http://kubernetes.io/docs/
52+
53+
[Service]
54+
ExecStart=/usr/bin/kubelet
55+
Restart=always
56+
StartLimitInterval=0
57+
RestartSec=10
58+
59+
[Install]
60+
WantedBy=multi-user.target
61+
`
62+
63+
var kubeadmRestoreTemplate = template.Must(template.New("kubeadmRestoreTemplate").Parse(`
64+
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
65+
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
66+
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
67+
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
68+
`))
69+
70+
var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))

0 commit comments

Comments
 (0)