diff --git a/Makefile b/Makefile index 9069ef5c7317..3ed6b81fef72 100755 --- a/Makefile +++ b/Makefile @@ -483,6 +483,13 @@ else docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) . endif +.PHONY: kic-base-image +kic-base-image: ## builds the base image used for kic. + docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true + docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) . + + + .PHONY: push-storage-provisioner-image push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud ifeq ($(GOARCH),amd64) @@ -591,6 +598,7 @@ out/mkcmp: out/performance-monitor: GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go + .PHONY: help help: @printf "\033[1mAvailable targets for minikube ${VERSION}\033[21m\n" diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index d8b8b594967d..4c99fa3c9644 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -232,7 +232,7 @@ func initNetworkingFlags() { startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.") - startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") + startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") startCmd.Flags().StringArrayVar(&dockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") startCmd.Flags().StringArrayVar(&dockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") } @@ -430,16 +430,22 @@ func displayEnviron(env []string) { } func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) { - addr, err := h.Driver.GetURL() - if err != nil { - exit.WithError("Failed to get driver URL", err) + addr := "" + var err error + if driver.IsKIC(h.DriverName) { + addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort))) + } else { + addr, err = h.Driver.GetURL() + if err != nil { + exit.WithError("Failed to get driver URL", err) + } + addr = strings.Replace(addr, "tcp://", "https://", -1) + addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1) } - addr = strings.Replace(addr, "tcp://", "https://", -1) - addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1) + if c.KubernetesConfig.APIServerName != constants.APIServerName { addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1) } - kcs := &kubeconfig.Settings{ ClusterName: clusterName, ClusterServerAddress: addr, @@ -980,29 +986,35 @@ func setDockerProxy() { } // autoSetDriverOptions sets the options needed for specific vm-driver automatically. -func autoSetDriverOptions(cmd *cobra.Command, drvName string) error { +func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) { + err = nil hints := driver.FlagDefaults(drvName) - if !cmd.Flags().Changed("extra-config") && hints.ExtraOptions != "" { - return extraOptions.Set(hints.ExtraOptions) + if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 { + for _, eo := range hints.ExtraOptions { + glog.Infof("auto setting extra-config to %q.", eo) + err = extraOptions.Set(eo) + if err != nil { + err = errors.Wrapf(err, "setting extra option %s", eo) + } + } } if !cmd.Flags().Changed(cacheImages) { viper.Set(cacheImages, hints.CacheImages) } - // currently only used for kic if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" { viper.Set(containerRuntime, hints.ContainerRuntime) - glog.Infof("auto set container runtime to %s for kic driver.", hints.ContainerRuntime) - + glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime) } - if !cmd.Flags().Changed("bootstrapper") && hints.Bootstrapper != "" { + + if !cmd.Flags().Changed(cmdcfg.Bootstrapper) && hints.Bootstrapper != "" { viper.Set(cmdcfg.Bootstrapper, hints.Bootstrapper) - glog.Infof("auto set bootstrapper to %s for kic driver.", hints.Bootstrapper) + glog.Infof("auto set %s to %q.", cmdcfg.Bootstrapper, hints.Bootstrapper) } - return nil + return err } // prepareNone prepares the user and host for the joy of the "none" driver diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile new file mode 100644 index 000000000000..9d4aed5e885f --- /dev/null +++ b/hack/images/kicbase.Dockerfile @@ -0,0 +1,18 @@ +ARG COMMIT_SHA +FROM kindest/node:v1.16.2 +USER root +RUN apt-get update && apt-get install -y \ + sudo \ + dnsutils \ + && apt-get clean -y +RUN rm -rf \ + /var/cache/debconf/* \ + /var/lib/apt/lists/* \ + /var/log/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/local/* \ + /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests +RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt" diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index cdc63bde6e39..e237c9befcf5 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -29,9 +29,19 @@ import ( "k8s.io/minikube/pkg/drivers/kic/node" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/constants" ) -// https://minikube.sigs.k8s.io/docs/reference/drivers/kic/ +// DefaultPodCIDR is The CIDR to be used for pods inside the node. +const DefaultPodCIDR = "10.244.0.0/16" + +// DefaultBindIPV4 is The default IP the container will bind to. +const DefaultBindIPV4 = "127.0.0.1" + +// BaseImage is the base image is used to spin up kic containers +const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db" + +// Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/ type Driver struct { *drivers.BaseDriver *pkgdrivers.CommonDriver @@ -43,16 +53,16 @@ type Driver struct { // Config is configuration for the kic driver used by registry type Config struct { - MachineName string // maps to the container name being created - CPU int // Number of CPU cores assigned to the container - Memory int // max memory in MB - StorePath string // lib machine store path - OCIBinary string // oci tool to use (docker, podman,...) - ImageDigest string // image name with sha to use for the node - APIServerPort int32 // port to connect to forward from container to user's machine - Mounts []oci.Mount // mounts - PortMappings []oci.PortMapping // container port mappings - Envs map[string]string // key,value of environment variables passed to the node + MachineName string // maps to the container name being created + CPU int // Number of CPU cores assigned to the container + Memory int // max memory in MB + StorePath string // libmachine store path + OCIBinary string // oci tool to use (docker, podman,...) + ImageDigest string // image name with sha to use for the node + HostBindPort int // port to connect to forward from container to user's machine + Mounts []oci.Mount // mounts + PortMappings []oci.PortMapping // container port mappings + Envs map[string]string // key,value of environment variables passed to the node } // NewDriver returns a fully configured Kic driver @@ -64,6 +74,7 @@ func NewDriver(c Config) *Driver { }, exec: command.NewKICRunner(c.MachineName, c.OCIBinary), NodeConfig: c, + OCIBinary: c.OCIBinary, } return d } @@ -77,15 +88,15 @@ func (d *Driver) Create() error { CPUs: strconv.Itoa(d.NodeConfig.CPU), Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb", Envs: d.NodeConfig.Envs, - ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)}, + ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)}, OCIBinary: d.NodeConfig.OCIBinary, } // control plane specific options params.PortMappings = append(params.PortMappings, oci.PortMapping{ ListenAddress: "127.0.0.1", - HostPort: d.NodeConfig.APIServerPort, - ContainerPort: 6443, + HostPort: int32(d.NodeConfig.HostBindPort), + ContainerPort: constants.APIServerPort, }) _, err := node.CreateNode(params) @@ -250,32 +261,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) { } return string(id), err } - -func ImageForVersion(ver string) (string, error) { - switch ver { - case "v1.11.10": - return "medyagh/kic:v1.11.10@sha256:23bb7f5e8dd2232ec829132172e87f7b9d8de65269630989e7dac1e0fe993b74", nil - case "v1.12.8": - return "medyagh/kic:v1.12.8@sha256:c74bc5f3efe3539f6e1ad7f11bf7c09f3091c0547cb28071f4e43067053e5898", nil - case "v1.12.9": - return "medyagh/kic:v1.12.9@sha256:ff82f58e18dcb22174e8eb09dae14f7edd82d91a83c7ef19e33298d0eba6a0e3", nil - case "v1.12.10": - return "medyagh/kic:v1.12.10@sha256:2d174bae7c20698e59791e7cca9b6db234053d1a92a009d5bb124e482540c70b", nil - case "v1.13.6": - return "medyagh/kic:v1.13.6@sha256:cf63e50f824fe17b90374d38d64c5964eb9fe6b3692669e1201fcf4b29af4964", nil - case "v1.13.7": - return "medyagh/kic:v1.13.7@sha256:1a6a5e1c7534cf3012655e99df680496df9bcf0791a304adb00617d5061233fa", nil - case "v1.14.3": - return "medyagh/kic:v1.14.3@sha256:cebec21f6af23d5dfa3465b88ddf4a1acb94c2c20a0a6ff8cc1c027b0a4e2cec", nil - case "v1.15.0": - return "medyagh/kic:v1.15.0@sha256:40d433d00a2837c8be829bd3cb0576988e377472062490bce0b18281c7f85303", nil - case "v1.15.3": - return "medyagh/kic:v1.15.3@sha256:f05ce52776a86c6ead806942d424de7076af3f115b0999332981a446329e6cf1", nil - case "v1.16.1": - return "medyagh/kic:v1.16.1@sha256:e74530d22e6a04442a97a09bdbba885ad693fcc813a0d1244da32666410d1ad1", nil - case "v1.16.2": - return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil - default: - return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil - } -} diff --git a/pkg/drivers/kic/node/node.go b/pkg/drivers/kic/node/node.go index acb8005a6ea1..6f1aa60c56ed 100644 --- a/pkg/drivers/kic/node/node.go +++ b/pkg/drivers/kic/node/node.go @@ -39,8 +39,8 @@ const ( // Node represents a handle to a kic node // This struct must be created by one of: CreateControlPlane type Node struct { - // must be one of docker container ID or name - name string + id string // container id + name string // container name r command.Runner // Runner ociBinary string } @@ -123,13 +123,15 @@ func CreateNode(p CreateConfig) (*Node, error) { // Find finds a node func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) { - _, err := oci.Inspect(ociBinary, name, "{{.Id}}") + n, err := oci.Inspect(ociBinary, name, "{{.Id}}") if err != nil { return nil, fmt.Errorf("can't find node %v", err) } return &Node{ - name: name, - r: cmder, + ociBinary: ociBinary, + id: n[0], + name: name, + r: cmder, }, nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 77235d73ee98..37859917cfcb 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -108,7 +108,6 @@ func Pause(ociBinary string, ociID string) error { // Inspect return low-level information on containers func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) { - cmd := exec.Command(ociBinary, "inspect", "-f", format, containerNameOrID) // ... against the "node" container @@ -359,7 +358,7 @@ func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]strin // 125 if err != nil { - return output, errors.Wrapf(err, "CreateContainer %v ", args) + return output, errors.Wrapf(err, "args: %v output: %s ", args, output) } return output, nil } diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 18b0c73c7fa9..6e35a79dc3d5 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -16,6 +16,11 @@ limitations under the License. package oci +const ( + Docker = "docker" + Podman = "podman" +) + /* These types are from https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183 diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 4252d7061758..a0b2cdc4e542 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -70,7 +70,7 @@ func GetCachedImageList(imageRepository string, version string, bootstrapper str case Kubeadm: return images.Kubeadm(imageRepository, version) case KIC: - return []string{"alpine"}, nil // for testing purpose just caching alpine for kicbs + return images.KIC(imageRepository, version) default: return []string{}, nil } diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go index bd59a8e9eb06..2b218a7373bb 100644 --- a/pkg/minikube/bootstrapper/bsutil/binaries.go +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil package will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go index d00c539baf61..0136fea42c90 100644 --- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/featuregates.go b/pkg/minikube/bootstrapper/bsutil/featuregates.go index 0f0c49092561..104dbf5a47fb 100644 --- a/pkg/minikube/bootstrapper/bsutil/featuregates.go +++ b/pkg/minikube/bootstrapper/bsutil/featuregates.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/featuregates_test.go b/pkg/minikube/bootstrapper/bsutil/featuregates_test.go index 3d3a4fb0c6c8..bdedff7e50cf 100644 --- a/pkg/minikube/bootstrapper/bsutil/featuregates_test.go +++ b/pkg/minikube/bootstrapper/bsutil/featuregates_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/files.go b/pkg/minikube/bootstrapper/bsutil/files.go index 7dff9a2dd46f..decb2fed55b8 100644 --- a/pkg/minikube/bootstrapper/bsutil/files.go +++ b/pkg/minikube/bootstrapper/bsutil/files.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( "path" + "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/vmpath" @@ -42,7 +43,6 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath, "0640"), assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile, "0644"), assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile, "0644"), - assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644"), } // Copy the default CNI config (k8s.conf), so that kubelet can successfully // start a Pod in the case a user hasn't manually installed any CNI plugin @@ -52,3 +52,33 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt } return fs } + +// AddAddons adds addons to list of files +func AddAddons(files *[]assets.CopyableFile, data interface{}) error { + // add addons to file list + // custom addons + if err := assets.AddMinikubeDirAssets(files); err != nil { + return errors.Wrap(err, "adding minikube dir assets") + } + // bundled addons + for _, addonBundle := range assets.Addons { + if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled { + for _, addon := range addonBundle.Assets { + if addon.IsTemplate() { + addonFile, err := addon.Evaluate(data) + if err != nil { + return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) + } + + *files = append(*files, addonFile) + } else { + *files = append(*files, addon) + } + } + } else if err != nil { + return nil + } + } + + return nil +} diff --git a/pkg/minikube/bootstrapper/bsutil/template/kubelet.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go similarity index 98% rename from pkg/minikube/bootstrapper/bsutil/template/kubelet.go rename to pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go index a35bf7d2e1e0..04c7c779c7ae 100644 --- a/pkg/minikube/bootstrapper/bsutil/template/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package template +package ktmpl import "text/template" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go new file mode 100644 index 000000000000..92f0ca3b13cf --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go @@ -0,0 +1,41 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import ( + "fmt" + "sort" +) + +// printMapInOrder sorts the keys and prints the map in order, combining key +// value pairs with the separator character +// +// Note: this is not necessary, but makes testing easy +func printMapInOrder(m map[string]string, sep string) []string { + if m == nil { + return nil + } + keys := []string{} + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for i, k := range keys { + keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k]) + } + return keys +} diff --git a/pkg/minikube/bootstrapper/bsutil/template/template_test.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go similarity index 98% rename from pkg/minikube/bootstrapper/bsutil/template/template_test.go rename to pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go index 4ac1ba3ccc8f..1b3782fdc3e9 100644 --- a/pkg/minikube/bootstrapper/bsutil/template/template_test.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package template +package ktmpl import ( "reflect" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go new file mode 100644 index 000000000000..2576bf07ffb8 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import "text/template" + +// V1Alpha1 is for Kubernetes v1.11 +var V1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{ + "printMapInOrder": printMapInOrder, +}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +{{if .NoTaintMaster}}noTaintMaster: true{{end}} +api: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} + controlPlaneEndpoint: localhost +kubernetesVersion: {{.KubernetesVersion}} +certificatesDir: {{.CertDir}} +networking: + serviceSubnet: {{.ServiceCIDR}} +etcd: + dataDir: {{.EtcdDataDir}} +nodeName: {{.NodeName}} +apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"] +{{if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}} +{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} + {{$val}}{{end}} +{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} + {{$i}}: {{$val}}{{end}} +{{end}}`)) diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go new file mode 100644 index 000000000000..3964e355a1a5 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go @@ -0,0 +1,69 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import "text/template" + +// V1Alpha3 is for Kubernetes v1.12 +var V1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{ + "printMapInOrder": printMapInOrder, +}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3 +kind: InitConfiguration +apiEndpoint: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} +bootstrapTokens: + - groups: + - system:bootstrappers:kubeadm:default-node-token + ttl: 24h0m0s + usages: + - signing + - authentication +nodeRegistration: + criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} + name: {{.NodeName}} + taints: [] +--- +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: ClusterConfiguration +{{if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} + {{$val}}{{end}} +{{end -}} +{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} + {{$i}}: {{$val}}{{end}} +{{end -}} +certificatesDir: {{.CertDir}} +clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"] +controlPlaneEndpoint: localhost:{{.APIServerPort}} +etcd: + local: + dataDir: {{.EtcdDataDir}} +kubernetesVersion: {{.KubernetesVersion}} +networking: + dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} + podSubnet: "{{ .PodSubnet }}" + serviceSubnet: {{.ServiceCIDR}} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +evictionHard: + nodefs.available: "0%" + nodefs.inodesFree: "0%" + imagefs.available: "0%" +`)) diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go new file mode 100644 index 000000000000..6b6c59a81b4c --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go @@ -0,0 +1,76 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import "text/template" + +// V1Beta1 is kubeadm config template for Kubernetes v1.13+ +var V1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{ + "printMapInOrder": printMapInOrder, +}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} +bootstrapTokens: + - groups: + - system:bootstrappers:kubeadm:default-node-token + ttl: 24h0m0s + usages: + - signing + - authentication +nodeRegistration: + criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} + name: {{.NodeName}} + taints: [] +--- +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +{{ if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{range .ExtraArgs}}{{.Component}}: + extraArgs: +{{- range $i, $val := printMapInOrder .Options ": " }} + {{$val}} +{{- end}} +{{end -}} +{{if .FeatureArgs}}featureGates: +{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}} +{{end -}}{{end -}} +certificatesDir: {{.CertDir}} +clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"] +controlPlaneEndpoint: localhost:{{.APIServerPort}} +dns: + type: CoreDNS +etcd: + local: + dataDir: {{.EtcdDataDir}} +kubernetesVersion: {{.KubernetesVersion}} +networking: + dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} + podSubnet: "{{.PodSubnet }}" + serviceSubnet: {{.ServiceCIDR}} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +imageGCHighThresholdPercent: 100 +evictionHard: + nodefs.available: "0%" + nodefs.inodesFree: "0%" + imagefs.available: "0%" +`)) diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index db580f3622f7..3887909c8317 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( @@ -24,12 +24,11 @@ import ( "github.com/blang/semver" "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/vmpath" - "k8s.io/minikube/pkg/util" ) // Container runtimes @@ -53,7 +52,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt return nil, errors.Wrap(err, "generating extra component config for kubeadm") } - // In case of no port assigned, use util.APIServerPort + // In case of no port assigned, use default nodePort := k8s.NodePort if nodePort <= 0 { nodePort = constants.APIServerPort @@ -76,7 +75,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt NoTaintMaster bool }{ CertDir: vmpath.GuestCertsDir, - ServiceCIDR: util.DefaultServiceCIDR, + ServiceCIDR: constants.DefaultServiceCIDR, PodSubnet: k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm), AdvertiseAddress: k8s.NodeIP, APIServerPort: nodePort, @@ -97,13 +96,13 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt opts.NoTaintMaster = true b := bytes.Buffer{} - configTmpl := template.KubeAdmConfigTmplV1Alpha1 + configTmpl := ktmpl.V1Alpha1 if version.GTE(semver.MustParse("1.12.0")) { - configTmpl = template.KubeAdmConfigTmplV1Alpha3 + configTmpl = ktmpl.V1Alpha3 } // v1beta1 works in v1.13, but isn't required until v1.14. if version.GTE(semver.MustParse("1.14.0-alpha.0")) { - configTmpl = template.KubeAdmConfigTmplV1Beta1 + configTmpl = ktmpl.V1Beta1 } if err := configTmpl.Execute(&b, opts); err != nil { return nil, err diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go index c0b90a2b4de9..b5fe8c7fe620 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go @@ -101,7 +101,7 @@ This test case has only 1 thing to test and that is the nnetworking/dnsDomain value */ func TestGenerateKubeadmYAMLDNS(t *testing.T) { - versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"} + versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12"} tests := []struct { name string runtime string diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index 48864d80feeb..5c21698deba2 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( @@ -22,7 +22,7 @@ import ( "path" "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" @@ -76,7 +76,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, ContainerRuntime: k8s.ContainerRuntime, KubeletPath: path.Join(binRoot(k8s.KubernetesVersion), "kubelet"), } - if err := template.KubeletSystemdTemplate.Execute(&b, opts); err != nil { + if err := ktmpl.KubeletSystemdTemplate.Execute(&b, opts); err != nil { return nil, err } @@ -87,7 +87,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) { var b bytes.Buffer opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")} - if err := template.KubeletServiceTemplate.Execute(&b, opts); err != nil { + if err := ktmpl.KubeletServiceTemplate.Execute(&b, opts); err != nil { return nil, errors.Wrap(err, "template execute") } return b.Bytes(), nil diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go index 090cab029040..3c93aa97c2d3 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go new file mode 100644 index 000000000000..40cb5c46783a --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -0,0 +1,141 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running kubernetes cluster is healthy +package kverify + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "os/exec" + "time" + + "github.com/docker/machine/libmachine/state" + "github.com/golang/glog" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/minikube/pkg/minikube/command" +) + +// APIServerProcess waits for api server to be healthy returns error if it doesn't +func APIServerProcess(runner command.Runner, start time.Time, timeout time.Duration) error { + glog.Infof("waiting for apiserver process to appear ...") + err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during process check") + } + rr, ierr := runner.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) + if ierr != nil { + glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command()) + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("apiserver process never appeared") + } + glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start)) + return nil +} + +// SystemPods verifies essential pods for running kurnetes is running +func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error { + glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port))) + pStart := time.Now() + podStart := time.Time{} + podList := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during pod check") + } + // Wait for any system pod, as waiting for apiserver may block until etcd + pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{}) + if len(pods.Items) < 2 { + podStart = time.Time{} + return false, nil + } + if err != nil { + podStart = time.Time{} + return false, nil + } + if podStart.IsZero() { + podStart = time.Now() + } + + glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart) + if time.Since(podStart) > 2*kconst.APICallRetryInterval { + glog.Infof("stability requirement met, returning") + return true, nil + } + return false, nil + } + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil { + return fmt.Errorf("apiserver never returned a pod list") + } + glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart)) + return nil +} + +// APIServerIsRunning waits for api server status to be running +func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Duration) error { + glog.Infof("waiting for apiserver healthz status ...") + hStart := time.Now() + healthz := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during healthz check") + } + + status, err := APIServerStatus(net.ParseIP(ip), port) + if err != nil { + glog.Warningf("status: %v", err) + return false, nil + } + if status != "Running" { + return false, nil + } + return true, nil + } + + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil { + return fmt.Errorf("apiserver healthz never reported healthy") + } + glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart)) + return nil +} + +// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State +func APIServerStatus(ip net.IP, apiserverPort int) (string, error) { + url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort))) + // To avoid: x509: certificate signed by unknown authority + tr := &http.Transport{ + Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set. + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + resp, err := client.Get(url) + // Connection refused, usually. + if err != nil { + return state.Stopped.String(), nil + } + if resp.StatusCode != http.StatusOK { + glog.Warningf("%s response: %v %+v", url, err, resp) + return state.Error.String(), nil + } + return state.Running.String(), nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/rbac.go b/pkg/minikube/bootstrapper/bsutil/rbac.go similarity index 56% rename from pkg/minikube/bootstrapper/kubeadm/rbac.go rename to pkg/minikube/bootstrapper/bsutil/rbac.go index 728c73163e42..12089bf6ab2d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/rbac.go +++ b/pkg/minikube/bootstrapper/bsutil/rbac.go @@ -14,10 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( + "fmt" "net" + "os/exec" + "strings" "time" "github.com/golang/glog" @@ -25,6 +28,7 @@ import ( rbac "k8s.io/api/rbac/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/util/retry" ) @@ -32,9 +36,9 @@ const ( rbacName = "minikube-rbac" ) -// elevateKubeSystemPrivileges gives the kube-system service account +// ElevateKubeSystemPrivileges gives the kube-system service account // cluster admin privileges to work with RBAC. -func elevateKubeSystemPrivileges(client kubernetes.Interface) error { +func ElevateKubeSystemPrivileges(client kubernetes.Interface) error { start := time.Now() clusterRoleBinding := &rbac.ClusterRoleBinding{ ObjectMeta: meta.ObjectMeta{ @@ -67,3 +71,31 @@ func elevateKubeSystemPrivileges(client kubernetes.Interface) error { glog.Infof("duration metric: took %s to wait for elevateKubeSystemPrivileges.", time.Since(start)) return nil } + +// AdjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config. +func AdjustResourceLimits(c command.Runner) error { + rr, err := c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj")) + if err != nil { + return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command()) + } + glog.Infof("apiserver oom_adj: %s", rr.Stdout.String()) + // oom_adj is already a negative number + if strings.HasPrefix(rr.Stdout.String(), "-") { + return nil + } + glog.Infof("adjusting apiserver oom_adj to -10") + + // Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster. + // It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS. + if _, err = c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil { + return errors.Wrap(err, fmt.Sprintf("oom_adj adjust")) + } + return nil +} + +// ExistingConfig checks if there are config files from possible previous kubernets cluster +func ExistingConfig(c command.Runner) error { + args := append([]string{"ls"}, expectedRemoteArtifacts...) + _, err := c.RunCmd(exec.Command("sudo", args...)) + return err +} diff --git a/pkg/minikube/bootstrapper/bsutil/template/template.go b/pkg/minikube/bootstrapper/bsutil/template/template.go deleted file mode 100644 index 92ac7975aec5..000000000000 --- a/pkg/minikube/bootstrapper/bsutil/template/template.go +++ /dev/null @@ -1,171 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package template - -import ( - "fmt" - "sort" - "text/template" -) - -// KubeAdmConfigTmplV1Alpha1 is for Kubernetes v1.11 -var KubeAdmConfigTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{ - "printMapInOrder": printMapInOrder, -}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1 -kind: MasterConfiguration -{{if .NoTaintMaster}}noTaintMaster: true{{end}} -api: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} - controlPlaneEndpoint: localhost -kubernetesVersion: {{.KubernetesVersion}} -certificatesDir: {{.CertDir}} -networking: - serviceSubnet: {{.ServiceCIDR}} -etcd: - dataDir: {{.EtcdDataDir}} -nodeName: {{.NodeName}} -{{if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}} -{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} - {{$val}}{{end}} -{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} - {{$i}}: {{$val}}{{end}} -{{end}}`)) - -// KubeAdmConfigTmplV1Alpha3 is for Kubernetes v1.12 -var KubeAdmConfigTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{ - "printMapInOrder": printMapInOrder, -}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3 -kind: InitConfiguration -apiEndpoint: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} -bootstrapTokens: - - groups: - - system:bootstrappers:kubeadm:default-node-token - ttl: 24h0m0s - usages: - - signing - - authentication -nodeRegistration: - criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} - name: {{.NodeName}} - taints: [] ---- -apiVersion: kubeadm.k8s.io/v1alpha3 -kind: ClusterConfiguration -{{if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} - {{$val}}{{end}} -{{end -}} -{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} - {{$i}}: {{$val}}{{end}} -{{end -}} -certificatesDir: {{.CertDir}} -clusterName: kubernetes -controlPlaneEndpoint: localhost:{{.APIServerPort}} -etcd: - local: - dataDir: {{.EtcdDataDir}} -kubernetesVersion: {{.KubernetesVersion}} -networking: - dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} - podSubnet: {{if .PodSubnet}}{{.PodSubnet}}{{else}}""{{end}} - serviceSubnet: {{.ServiceCIDR}} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -evictionHard: - nodefs.available: "0%" - nodefs.inodesFree: "0%" - imagefs.available: "0%" -`)) - -// KubeAdmConfigTmplV1Beta1 is for Kubernetes v1.13+ -var KubeAdmConfigTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{ - "printMapInOrder": printMapInOrder, -}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} -bootstrapTokens: - - groups: - - system:bootstrappers:kubeadm:default-node-token - ttl: 24h0m0s - usages: - - signing - - authentication -nodeRegistration: - criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} - name: {{.NodeName}} - taints: [] ---- -apiVersion: kubeadm.k8s.io/v1beta1 -kind: ClusterConfiguration -{{ if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{range .ExtraArgs}}{{.Component}}: - extraArgs: -{{- range $i, $val := printMapInOrder .Options ": " }} - {{$val}} -{{- end}} -{{end -}} -{{if .FeatureArgs}}featureGates: -{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}} -{{end -}}{{end -}} -certificatesDir: {{.CertDir}} -clusterName: kubernetes -controlPlaneEndpoint: localhost:{{.APIServerPort}} -dns: - type: CoreDNS -etcd: - local: - dataDir: {{.EtcdDataDir}} -kubernetesVersion: {{.KubernetesVersion}} -networking: - dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} - podSubnet: "" - serviceSubnet: {{.ServiceCIDR}} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -imageGCHighThresholdPercent: 100 -evictionHard: - nodefs.available: "0%" - nodefs.inodesFree: "0%" - imagefs.available: "0%" -`)) - -// printMapInOrder sorts the keys and prints the map in order, combining key -// value pairs with the separator character -// -// Note: this is not necessary, but makes testing easy -func printMapInOrder(m map[string]string, sep string) []string { - if m == nil { - return nil - } - keys := []string{} - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - for i, k := range keys { - keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k]) - } - return keys -} diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml index cb5e0e62e5a5..e44ab4c92aba 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /run/containerd/containerd.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml index a78261c550ed..ed91117f7308 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /run/containerd/containerd.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml index a78261c550ed..ed91117f7308 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /run/containerd/containerd.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml index 15bb16aaa158..c15e61950852 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /var/run/crio/crio.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml index 7e33c3c86198..eb4fd3fd1cfe 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /var/run/crio/crio.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml index 13ea3d1c53d9..8e3e566bc215 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml @@ -12,5 +12,6 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml index 5e121b085d19..6e0860426320 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] imageRepository: test/repo apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml index aac8924cbe9b..a3d5c8230327 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" fail-no-swap: "true" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml index 2c7ad8fc47c6..fee7f3684f18 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml index c13a251804da..c3399153f923 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: @@ -28,7 +29,7 @@ etcd: kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local - podSubnet: 192.168.32.0/20 + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml index 1398527e91fe..9beec13a4d53 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml index d33f0e13dc62..88253bcb486d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml @@ -29,6 +29,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml index c076b8ad9f59..4a3555c9afe5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml index 36856509b97e..7ccd2edf2854 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml index a4a813404c50..96bf607d6551 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml index dab9a37c2353..f0b48f62e13d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml @@ -22,6 +22,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml index 3a6ff1aeed2e..22809c5ce969 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml @@ -26,6 +26,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml index f365193c6170..3106e329d1f6 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml index d5f799e56fb9..279a7c20a381 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: @@ -28,7 +29,7 @@ etcd: kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local - podSubnet: 192.168.32.0/20 + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml index 806106b3b7d1..6994f8d92b8a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml index e602810b0d49..1304deaa3e5b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml @@ -29,6 +29,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml index b82b3bf69bb4..a333a2539986 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml index 9995a46c1417..afc4d8a8215b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml index cdf2a0405fe0..89d56b7bfddc 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml index b2b8b6e3fb9d..3fc3446ce482 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml @@ -22,6 +22,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml index 484dd5a352b4..9a4466840e80 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml @@ -26,6 +26,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml index 5755f5641d6b..6d4eeeeba358 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml index f4ff177f6757..3c6997ddecb0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml index f4ff177f6757..d96e27170cbb 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml index 6f46996151b4..1ad22d184b96 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml index 47cfd0ecbb79..062233c224e8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml index 57371bf704a4..bedb175306a9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml index 93ce010c2ac4..dc8df12a4440 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml index 2c1a895bca58..c9c6846a3813 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml index 325265c2c6a3..2d0c7e807a94 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml index 9f08022e331d..30bb67574a6b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml index 30e3b200f136..4fc81aa1c51b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml index 30e3b200f136..5fcd575dee90 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml index bcc3e64b0030..1d82fb284d6d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml index 7cc895ca1583..eadb7a3fcdb2 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml index f31d94301338..5e9b1e4bfe00 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml index d6474d3bee3f..e8e9158ad2e5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml index aec5820a00a3..1362db1f37c0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml index eb354d47ee50..31462f323fa2 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml index d7e9c9e97596..1befae1db7b1 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml index d4de91b7eb07..65ead231372c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml index d4de91b7eb07..3b269defb341 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml index 5aa8ca8652ac..b04f318a0866 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml index 57ac32b75eba..c64f2a1cba2f 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml index 4e1949e0f26f..60d7ae8eea10 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml index 1fd0562a5590..08c7528ce35a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml index f1b08545b9b2..62bf5372ac57 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml index 6c02b9c29e2d..1c0d76eba4d5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml index 166ebbbbe8e6..872fa683e0c8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml index 6bb42a82e8e6..7336030107d8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.17.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml index 6bb42a82e8e6..1f5fb46836cd 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml index 6cc338fb211f..fe876ff8884d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml index 0fddd573cb28..6704dc431b93 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml index 53cbaba48837..263492d5bece 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml index e32a188927f3..7441511cb0ff 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml index 1dd863df2843..c746585e4dd1 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml index 03b1bb3c7205..d47790e95ea0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/types.go b/pkg/minikube/bootstrapper/bsutil/types.go new file mode 100644 index 000000000000..a436fdf85c0b --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/types.go @@ -0,0 +1,29 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one +package bsutil + +// SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use. +var SkipAdditionalPreflights = map[string][]string{} + +// expectedRemoteArtifacts remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. +// NOTE: /etc is not persistent across restarts, so don't bother checking there +var expectedRemoteArtifacts = []string{ + "/var/lib/kubelet/kubeadm-flags.env", + "/var/lib/kubelet/config.yaml", + EtcdDataDir(), +} diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index d0731b24cd01..21c573454cfa 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/tools/clientcmd/api/latest" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -110,7 +111,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { kcs := &kubeconfig.Settings{ ClusterName: k8s.NodeName, - ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort), + ClusterServerAddress: fmt.Sprintf("https://%s", net.JoinHostPort("localhost", fmt.Sprint(k8s.NodePort))), ClientCertificate: path.Join(vmpath.GuestCertsDir, "apiserver.crt"), ClientKey: path.Join(vmpath.GuestCertsDir, "apiserver.key"), CertificateAuthority: path.Join(vmpath.GuestCertsDir, "ca.crt"), @@ -175,7 +176,7 @@ func generateCerts(k8s config.KubernetesConfig) error { apiServerIPs := append( k8s.APIServerIPs, - []net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP("10.0.0.1")}...) + []net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP(kic.DefaultBindIPV4), net.ParseIP("10.0.0.1")}...) apiServerNames := append(k8s.APIServerNames, k8s.APIServerName) apiServerAlternateNames := append( apiServerNames, diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 7d65e6c3af38..167cf4084e62 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -38,7 +38,7 @@ func TestSetupCerts(t *testing.T) { k8s := config.KubernetesConfig{ APIServerName: constants.APIServerName, DNSDomain: constants.ClusterDNSDomain, - ServiceCIDR: util.DefaultServiceCIDR, + ServiceCIDR: constants.DefaultServiceCIDR, } if err := os.Mkdir(filepath.Join(tempDir, "certs"), 0777); err != nil { diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go index b80e17f372c0..80a06f385e7b 100644 --- a/pkg/minikube/bootstrapper/images/images.go +++ b/pkg/minikube/bootstrapper/images/images.go @@ -18,20 +18,99 @@ limitations under the License. package images import ( + "fmt" "path" "runtime" + + "github.com/blang/semver" ) -// ArchTag returns a CPU architecture suffix for images -func ArchTag(hasTag bool) string { +// Pause returns the image name to pull for the pause image +func Pause(mirror string) string { + // Should match `PauseVersion` in: + // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + return path.Join(kubernetesRepo(mirror), "pause"+archTag(false)+"3.1") +} + +// essentials returns images needed too bootstrap a kubenretes +func essentials(mirror string, v semver.Version) []string { + imgs := []string{ + componentImage("kube-proxy", v, mirror), + componentImage("kube-scheduler", v, mirror), + componentImage("kube-controller-manager", v, mirror), + componentImage("kube-apiserver", v, mirror), + coreDNS(v, mirror), + etcd(v, mirror), + Pause(mirror), + } + return imgs +} + +// componentImage returns a Kubernetes component image to pull +func componentImage(name string, v semver.Version, mirror string) string { + needsArchSuffix := false + ancient := semver.MustParseRange("<1.12.0") + if ancient(v) { + needsArchSuffix = true + } + + return fmt.Sprintf("%sv%s", path.Join(kubernetesRepo(mirror), name+archTag(needsArchSuffix)), v) +} + +// coreDNS returns the images used for CoreDNS +func coreDNS(v semver.Version, mirror string) string { + // Should match `CoreDNSVersion` in + // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + cv := "1.6.5" + switch v.Minor { + case 16: + cv = "1.6.2" + case 15, 14: + cv = "1.3.1" + case 13: + cv = "1.2.6" + case 12: + cv = "1.2.2" + case 11: + cv = "1.1.3" + } + return path.Join(kubernetesRepo(mirror), "coredns"+archTag(false)+cv) +} + +// etcd returns the image used for etcd +func etcd(v semver.Version, mirror string) string { + needsArchSuffix := false + ancient := semver.MustParseRange("<1.12.0") + if ancient(v) { + needsArchSuffix = true + } + + // Should match `DefaultEtcdVersion` in: + // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + ev := "3.4.3-0" + switch v.Minor { + case 16: + ev = "3.3.15-0" + case 14, 15: + ev = "3.3.10" + case 12, 13: + ev = "3.2.24" + case 11: + ev = "3.2.18" + } + return path.Join(kubernetesRepo(mirror), "etcd"+archTag(needsArchSuffix)+ev) +} + +// archTag returns a CPU architecture suffix for images +func archTag(hasTag bool) string { if runtime.GOARCH == "amd64" && !hasTag { return ":" } return "-" + runtime.GOARCH + ":" } -// Auxiliary returns images that are helpful for running minikube -func Auxiliary(mirror string) []string { +// auxiliary returns images that are helpful for running minikube +func auxiliary(mirror string) []string { return []string{ addonManager(mirror), storageProvisioner(mirror), @@ -40,21 +119,14 @@ func Auxiliary(mirror string) []string { } } -// Pause returns the image name to pull for the pause image -func Pause(mirror string) string { - // Should match `PauseVersion` in: - // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - return path.Join(KubernetesRepo(mirror), "pause"+ArchTag(false)+"3.1") -} - // storageProvisioner returns the minikube storage provisioner image func storageProvisioner(mirror string) string { - return path.Join(minikubeRepo(mirror), "storage-provisioner"+ArchTag(false)+"v1.8.1") + return path.Join(minikubeRepo(mirror), "storage-provisioner"+archTag(false)+"v1.8.1") } // addonManager returns the Kubernetes addon manager image func addonManager(mirror string) string { - return path.Join(KubernetesRepo(mirror), "kube-addon-manager"+ArchTag(false)+"v9.0.2") + return path.Join(kubernetesRepo(mirror), "kube-addon-manager"+archTag(false)+"v9.0.2") } // dashboardFrontend returns the image used for the dashboard frontend diff --git a/pkg/minikube/bootstrapper/images/images_test.go b/pkg/minikube/bootstrapper/images/images_test.go index 31a487e4c4f0..3d12adc59131 100644 --- a/pkg/minikube/bootstrapper/images/images_test.go +++ b/pkg/minikube/bootstrapper/images/images_test.go @@ -29,7 +29,7 @@ func TestAuxiliary(t *testing.T) { "kubernetesui/dashboard:v2.0.0-beta8", "kubernetesui/metrics-scraper:v1.0.2", } - got := Auxiliary("") + got := auxiliary("") if diff := cmp.Diff(want, got); diff != "" { t.Errorf("images mismatch (-want +got):\n%s", diff) } @@ -42,7 +42,7 @@ func TestAuxiliaryMirror(t *testing.T) { "test.mirror/dashboard:v2.0.0-beta8", "test.mirror/metrics-scraper:v1.0.2", } - got := Auxiliary("test.mirror") + got := auxiliary("test.mirror") if diff := cmp.Diff(want, got); diff != "" { t.Errorf("images mismatch (-want +got):\n%s", diff) } diff --git a/pkg/minikube/bootstrapper/images/kic.go b/pkg/minikube/bootstrapper/images/kic.go new file mode 100644 index 000000000000..9d60fab6fa47 --- /dev/null +++ b/pkg/minikube/bootstrapper/images/kic.go @@ -0,0 +1,35 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package images + +import ( + "strings" + + "github.com/blang/semver" + "github.com/pkg/errors" +) + +// KIC returns a list of images necessary to bootstrap kic +// it is same as vm-kubeadm minus auxulary +func KIC(mirror string, version string) ([]string, error) { + v, err := semver.Make(strings.TrimPrefix(version, "v")) + if err != nil { + return nil, errors.Wrap(err, "semver") + } + imgs := essentials(mirror, v) + return imgs, nil +} diff --git a/pkg/minikube/bootstrapper/images/kubeadm.go b/pkg/minikube/bootstrapper/images/kubeadm.go index 7f1071a3f30f..2c3ea23b626c 100644 --- a/pkg/minikube/bootstrapper/images/kubeadm.go +++ b/pkg/minikube/bootstrapper/images/kubeadm.go @@ -17,8 +17,6 @@ limitations under the License. package images import ( - "fmt" - "path" "strings" "github.com/blang/semver" @@ -31,71 +29,7 @@ func Kubeadm(mirror string, version string) ([]string, error) { if err != nil { return nil, errors.Wrap(err, "semver") } - - imgs := []string{ - componentImage("kube-proxy", v, mirror), - componentImage("kube-scheduler", v, mirror), - componentImage("kube-controller-manager", v, mirror), - componentImage("kube-apiserver", v, mirror), - coreDNS(v, mirror), - etcd(v, mirror), - Pause(mirror), - } - imgs = append(imgs, Auxiliary(mirror)...) + imgs := essentials(mirror, v) + imgs = append(imgs, auxiliary(mirror)...) return imgs, nil } - -// componentImage returns a Kubernetes component image to pull -func componentImage(name string, v semver.Version, mirror string) string { - needsArchSuffix := false - ancient := semver.MustParseRange("<1.12.0") - if ancient(v) { - needsArchSuffix = true - } - - return fmt.Sprintf("%sv%s", path.Join(KubernetesRepo(mirror), name+ArchTag(needsArchSuffix)), v) -} - -// coreDNS returns the images used for CoreDNS -func coreDNS(v semver.Version, mirror string) string { - // Should match `CoreDNSVersion` in - // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - cv := "1.6.5" - switch v.Minor { - case 16: - cv = "1.6.2" - case 15, 14: - cv = "1.3.1" - case 13: - cv = "1.2.6" - case 12: - cv = "1.2.2" - case 11: - cv = "1.1.3" - } - return path.Join(KubernetesRepo(mirror), "coredns"+ArchTag(false)+cv) -} - -// etcd returns the image used for etcd -func etcd(v semver.Version, mirror string) string { - needsArchSuffix := false - ancient := semver.MustParseRange("<1.12.0") - if ancient(v) { - needsArchSuffix = true - } - - // Should match `DefaultEtcdVersion` in: - // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - ev := "3.4.3-0" - switch v.Minor { - case 16: - ev = "3.3.15-0" - case 14, 15: - ev = "3.3.10" - case 12, 13: - ev = "3.2.24" - case 11: - ev = "3.2.18" - } - return path.Join(KubernetesRepo(mirror), "etcd"+ArchTag(needsArchSuffix)+ev) -} diff --git a/pkg/minikube/bootstrapper/images/repo.go b/pkg/minikube/bootstrapper/images/repo.go index 02cf44345b2f..ca9e3c366a86 100644 --- a/pkg/minikube/bootstrapper/images/repo.go +++ b/pkg/minikube/bootstrapper/images/repo.go @@ -19,8 +19,8 @@ package images // DefaultKubernetesRepo is the default Kubernetes repository const DefaultKubernetesRepo = "k8s.gcr.io" -// KubernetesRepo returns the official Kubernetes repository, or an alternate -func KubernetesRepo(mirror string) string { +// kubernetesRepo returns the official Kubernetes repository, or an alternate +func kubernetesRepo(mirror string) string { if mirror != "" { return mirror } diff --git a/pkg/minikube/bootstrapper/kicbs/cni.go b/pkg/minikube/bootstrapper/kicbs/cni.go index 44eb980ac7cd..caa4534b47d6 100644 --- a/pkg/minikube/bootstrapper/kicbs/cni.go +++ b/pkg/minikube/bootstrapper/kicbs/cni.go @@ -14,12 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bootstrapper for kic +// Package kicbs bootstrapper for kic package kicbs const defaultCNIManifest = ` -# kindnetd networking manifest -# would you kindly template this file --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -99,7 +97,7 @@ spec: fieldRef: fieldPath: status.podIP - name: POD_SUBNET - value: {{ .PodSubnet }} + value: 10.244.0.0/16 volumeMounts: - name: cni-cfg mountPath: /etc/cni/net.d diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go index c613d35e6823..8e668587d302 100644 --- a/pkg/minikube/bootstrapper/kicbs/kicbs.go +++ b/pkg/minikube/bootstrapper/kicbs/kicbs.go @@ -14,21 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bootstrapper for kic +// Package kicbs bootstrapper for kic package kicbs import ( "fmt" "net" "os/exec" + "strings" "time" + "github.com/blang/semver" "github.com/docker/machine/libmachine" "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" + "k8s.io/client-go/kubernetes" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -36,11 +43,13 @@ import ( "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/vmpath" ) // Bootstrapper is a bootstrapper using kicbs type Bootstrapper struct { c command.Runner + k8sClient *kubernetes.Clientset // kubernetes client used to verify pods inside cluster contextName string } @@ -60,9 +69,9 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) { // UpdateCluster updates the cluster func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { - images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) + images, err := images.KIC(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) if err != nil { - return errors.Wrap(err, "kubeadm images") + return errors.Wrap(err, "kic images") } if cfg.KubernetesConfig.ShouldLoadCachedImages { @@ -101,15 +110,15 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { return errors.Wrap(err, "downloading binaries") } - var cniFile []byte = nil - if cfg.KubernetesConfig.EnableDefaultCNI { - cniFile = []byte(defaultCNIManifest) - } + cniFile := []byte(defaultCNIManifest) + files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) - // if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { + // TODO: add addons for kic later + // if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { // return errors.Wrap(err, "adding addons") // } + for _, f := range files { if err := k.c.Copy(f); err != nil { return errors.Wrapf(err, "copy") @@ -122,28 +131,216 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { return nil } -func (k *Bootstrapper) PullImages(config.KubernetesConfig) error { - return fmt.Errorf("the PullImages is not implemented in kicbs yet") +// SetupCerts generates the certs the cluster +func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error { + return bootstrapper.SetupCerts(k.c, cfg) } -func (k *Bootstrapper) StartCluster(config.KubernetesConfig) error { - return fmt.Errorf("the StartCluster is not implemented in kicbs yet") + +// PullImages downloads images that will be used by Kubernetes +func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return errors.Wrap(err, "parsing kubernetes version") + } + if version.LT(semver.MustParse("1.11.0")) { + return fmt.Errorf("pull command is not supported by kubeadm v%s", version) + } + + rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath))) + if err != nil { + return errors.Wrapf(err, "running cmd: %q", rr.Command()) + } + return nil +} + +// StartCluster starts the cluster +func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { + k8s.NodeIP = kic.DefaultBindIPV4 + err := bsutil.ExistingConfig(k.c) + if err == nil { // if there is an existing cluster don't reconfigure it + return k.restartCluster(k8s) + } + glog.Infof("existence check: %v", err) + + start := time.Now() + glog.Infof("StartCluster: %+v", k8s) + defer func() { + glog.Infof("StartCluster complete in %s", time.Since(start)) + }() + + extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) + r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) + if err != nil { + return err + } + + ignore := []string{ + fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(bsutil.EtcdDataDir(), "/", "-", -1)), + "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", + "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", + "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml", + "FileAvailable--etc-kubernetes-manifests-etcd.yaml", + "FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", // for kic only + "Port-10250", // For "none" users who already have a kubelet online + "Swap", // For "none" users who have swap configured + "SystemVerification", // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64" + } + ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) + + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ","))) + glog.Infof("starting kubeadm init") + if rr, err := k.c.RunCmd(c); err != nil { + return errors.Wrapf(err, "init failed. cmd: %q output: %q", rr.Command(), rr.Output()) + } + + glog.Infof("applying kic overlay network") + if err := k.applyOverlayNetwork(); err != nil { + return errors.Wrap(err, "applying kic overlay network") + } + + glog.Infof("Skipping Configuring cluster permissions for kic...") + + if err := bsutil.AdjustResourceLimits(k.c); err != nil { + glog.Warningf("unable to adjust resource limits: %v", err) + } + + return nil +} + +// restartCluster restarts the Kubernetes cluster configured by kubeadm +func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { + glog.Infof("restartCluster start") + + start := time.Now() + defer func() { + glog.Infof("restartCluster took %s", time.Since(start)) + }() + + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return errors.Wrap(err, "parsing kubernetes version") + } + + phase := "alpha" + controlPlane := "controlplane" + if version.GTE(semver.MustParse("1.13.0")) { + phase = "init" + controlPlane = "control-plane" + } + + baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase) + cmds := []string{ + fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath), + fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath), + fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, bsutil.KubeadmYamlPath), + fmt.Sprintf("%s phase etcd local --config %s", baseCmd, bsutil.KubeadmYamlPath), + } + + // Run commands one at a time so that it is easier to root cause failures. + for _, c := range cmds { + rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c)) + if err != nil { + return errors.Wrapf(err, "running cmd: %s", rr.Command()) + } + } + + // We must ensure that the apiserver is healthy before proceeding + if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil { + return errors.Wrap(err, "apiserver healthz") + } + + client, err := k.client(k8s) + if err != nil { + return errors.Wrap(err, "getting k8s client") + } + + if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { + return errors.Wrap(err, "system pods") + } + + // Explicitly re-enable kubeadm addons (proxy, coredns) so that they will check for IP or configuration changes. + if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, bsutil.KubeadmYamlPath))); err != nil { + return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command())) + } + + if err := bsutil.AdjustResourceLimits(k.c); err != nil { + glog.Warningf("unable to adjust resource limits: %v", err) + } + return nil +} + +// WaitForCluster blocks until the cluster appears to be healthy +func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { + start := time.Now() + out.T(out.Waiting, "Waiting for cluster to come online ...") + if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { + return errors.Wrap(err, "wait for api proc") + } + + if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil { + return err + } + + c, err := k.client(k8s) // getting kubernetes client before polling. + if err != nil { + return errors.Wrap(err, "get k8s client") + } + + if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil { + return errors.Wrap(err, "wait for system pods") + } + + return nil } func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error { return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet") } -func (k *Bootstrapper) WaitForCluster(config.KubernetesConfig, time.Duration) error { - return fmt.Errorf("the WaitForCluster is not implemented in kicbs yet") -} + func (k *Bootstrapper) LogCommands(bootstrapper.LogOptions) map[string]string { return map[string]string{} } -func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error { - return fmt.Errorf("the SetupCerts is not implemented in kicbs yet") -} + func (k *Bootstrapper) GetKubeletStatus() (string, error) { return "", fmt.Errorf("the GetKubeletStatus is not implemented in kicbs yet") } func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) { return "", fmt.Errorf("the GetAPIServerStatus is not implemented in kicbs yet") } + +// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver +func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { + if k.k8sClient != nil { + return k.k8sClient, nil + } + + config, err := kapi.ClientConfig(k.contextName) + if err != nil { + return nil, errors.Wrap(err, "client config") + } + + endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort))) + if config.Host != endpoint { + glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) + config.Host = endpoint + } + c, err := kubernetes.NewForConfig(config) + if err == nil { + k.k8sClient = c + } + return c, err +} + +// applyOverlayNetwork applies the CNI plugin needed to make kic work +func (k *Bootstrapper) applyOverlayNetwork() error { + cmd := exec.Command( + "kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf", + "-f", bsutil.DefaultCNIConfigPath, + ) + if rr, err := k.c.RunCmd(cmd); err != nil { + return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output()) + } + return nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 51f954d8a55e..f5cfe92dc367 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -17,12 +17,10 @@ limitations under the License. package kubeadm import ( - "crypto/tls" "os/exec" "fmt" "net" - "net/http" // WARNING: Do not use path/filepath in this package unless you want bizarre Windows paths @@ -36,14 +34,13 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -55,20 +52,10 @@ import ( "k8s.io/minikube/pkg/util/retry" ) -// remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. -// NOTE: /etc is not persistent across restarts, so don't bother checking there -var expectedArtifacts = []string{ - "/var/lib/kubelet/kubeadm-flags.env", - "/var/lib/kubelet/config.yaml", - bsutil.EtcdDataDir(), -} - -// SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use. -var SkipAdditionalPreflights = map[string][]string{} - // Bootstrapper is a bootstrapper using kubeadm type Bootstrapper struct { c command.Runner + k8sClient *kubernetes.Clientset // kubernetes client used to verify pods inside cluster contextName string } @@ -83,7 +70,7 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) { if err != nil { return nil, errors.Wrap(err, "command runner") } - return &Bootstrapper{c: runner, contextName: name}, nil + return &Bootstrapper{c: runner, contextName: name, k8sClient: nil}, nil } // GetKubeletStatus returns the kubelet status @@ -106,23 +93,7 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) { // GetAPIServerStatus returns the api-server status func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) { - url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), strconv.Itoa(apiserverPort))) - // To avoid: x509: certificate signed by unknown authority - tr := &http.Transport{ - Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set. - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - resp, err := client.Get(url) - // Connection refused, usually. - if err != nil { - return state.Stopped.String(), nil - } - if resp.StatusCode != http.StatusOK { - glog.Warningf("%s response: %v %+v", url, err, resp) - return state.Error.String(), nil - } - return state.Running.String(), nil + return kverify.APIServerStatus(ip, apiserverPort) } // LogCommands returns a map of log type to a command which will display that log. @@ -167,16 +138,10 @@ func (k *Bootstrapper) createCompatSymlinks() error { return nil } -func (k *Bootstrapper) existingConfig() error { - args := append([]string{"ls"}, expectedArtifacts...) - _, err := k.c.RunCmd(exec.Command("sudo", args...)) - return err -} - // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { - err := k.existingConfig() - if err == nil { + err := bsutil.ExistingConfig(k.c) + if err == nil { // if there is an existing cluster don't reconfigure it return k.restartCluster(k8s) } glog.Infof("existence check: %v", err) @@ -209,7 +174,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { "Port-10250", // For "none" users who already have a kubelet online "Swap", // For "none" users who have swap configured } - ignore = append(ignore, SkipAdditionalPreflights[r.Name()]...) + ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) // Allow older kubeadm versions to function with newer Docker releases. if version.LT(semver.MustParse("1.13.0")) { @@ -223,79 +188,31 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { } glog.Infof("Configuring cluster permissions ...") - elevate := func() error { client, err := k.client(k8s) if err != nil { return err } - return elevateKubeSystemPrivileges(client) + return bsutil.ElevateKubeSystemPrivileges(client) } if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil { return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") } - if err := k.adjustResourceLimits(); err != nil { + if err := bsutil.AdjustResourceLimits(k.c); err != nil { glog.Warningf("unable to adjust resource limits: %v", err) } return nil } -// adjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config. -func (k *Bootstrapper) adjustResourceLimits() error { - rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj")) - if err != nil { - return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command()) - } - glog.Infof("apiserver oom_adj: %s", rr.Stdout.String()) - // oom_adj is already a negative number - if strings.HasPrefix(rr.Stdout.String(), "-") { - return nil - } - glog.Infof("adjusting apiserver oom_adj to -10") - - // Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster. - // It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS. - if _, err = k.c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil { - return errors.Wrap(err, fmt.Sprintf("oom_adj adjust")) - } - - return nil -} - -func addAddons(files *[]assets.CopyableFile, data interface{}) error { - // add addons to file list - // custom addons - if err := assets.AddMinikubeDirAssets(files); err != nil { - return errors.Wrap(err, "adding minikube dir assets") - } - // bundled addons - for _, addonBundle := range assets.Addons { - if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled { - for _, addon := range addonBundle.Assets { - if addon.IsTemplate() { - addonFile, err := addon.Evaluate(data) - if err != nil { - return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) - } - - *files = append(*files, addonFile) - } else { - *files = append(*files, addon) - } - } - } else if err != nil { - return nil - } +// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver +func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { + if k.k8sClient != nil { + return k.k8sClient, nil } - return nil -} - -// client returns a Kubernetes client to use to speak to a kubeadm launched apiserver -func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { config, err := kapi.ClientConfig(k.contextName) if err != nil { return nil, errors.Wrap(err, "client config") @@ -306,108 +223,30 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) config.Host = endpoint } - - return kubernetes.NewForConfig(config) -} - -func (k *Bootstrapper) waitForAPIServerProcess(start time.Time, timeout time.Duration) error { - glog.Infof("waiting for apiserver process to appear ...") - err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) { - if time.Since(start) > timeout { - return false, fmt.Errorf("cluster wait timed out during process check") - } - rr, ierr := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) - if ierr != nil { - glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command()) - return false, nil - } - return true, nil - }) - if err != nil { - return fmt.Errorf("apiserver process never appeared") - } - glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start)) - return nil -} - -func (k *Bootstrapper) waitForAPIServerHealthz(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error { - glog.Infof("waiting for apiserver healthz status ...") - hStart := time.Now() - healthz := func() (bool, error) { - if time.Since(start) > timeout { - return false, fmt.Errorf("cluster wait timed out during healthz check") - } - - status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort) - if err != nil { - glog.Warningf("status: %v", err) - return false, nil - } - if status != "Running" { - return false, nil - } - return true, nil - } - - if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil { - return fmt.Errorf("apiserver healthz never reported healthy") - } - glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart)) - return nil -} - -func (k *Bootstrapper) waitForSystemPods(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error { - glog.Infof("waiting for kube-system pods to appear ...") - pStart := time.Now() - client, err := k.client(k8s) - if err != nil { - return errors.Wrap(err, "client") - } - - podStart := time.Time{} - podList := func() (bool, error) { - if time.Since(start) > timeout { - return false, fmt.Errorf("cluster wait timed out during pod check") - } - // Wait for any system pod, as waiting for apiserver may block until etcd - pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{}) - if len(pods.Items) < 2 { - podStart = time.Time{} - return false, nil - } - if err != nil { - podStart = time.Time{} - return false, nil - } - if podStart.IsZero() { - podStart = time.Now() - } - - glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart) - if time.Since(podStart) > 2*kconst.APICallRetryInterval { - glog.Infof("stability requirement met, returning") - return true, nil - } - return false, nil - } - if err = wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil { - return fmt.Errorf("apiserver never returned a pod list") + c, err := kubernetes.NewForConfig(config) + if err == nil { + k.k8sClient = c } - glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart)) - return nil + return c, err } // WaitForCluster blocks until the cluster appears to be healthy func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { start := time.Now() out.T(out.Waiting, "Waiting for cluster to come online ...") - if err := k.waitForAPIServerProcess(start, timeout); err != nil { + if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { return err } - if err := k.waitForAPIServerHealthz(start, k8s, timeout); err != nil { + if err := kverify.APIServerIsRunning(start, k8s.NodeIP, k8s.NodePort, timeout); err != nil { return err } - return k.waitForSystemPods(start, k8s, timeout) + + c, err := k.client(k8s) + if err != nil { + return errors.Wrap(err, "get k8s client") + } + + return kverify.SystemPods(c, start, k8s.NodeIP, k8s.NodePort, timeout) } // restartCluster restarts the Kubernetes cluster configured by kubeadm @@ -452,10 +291,16 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { } // We must ensure that the apiserver is healthy before proceeding - if err := k.waitForAPIServerHealthz(time.Now(), k8s, kconst.DefaultControlPlaneTimeout); err != nil { + if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil { return errors.Wrap(err, "apiserver healthz") } - if err := k.waitForSystemPods(time.Now(), k8s, kconst.DefaultControlPlaneTimeout); err != nil { + + client, err := k.client(k8s) + if err != nil { + return errors.Wrap(err, "getting k8s client") + } + + if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { return errors.Wrap(err, "system pods") } @@ -464,7 +309,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command())) } - if err := k.adjustResourceLimits(); err != nil { + if err := bsutil.AdjustResourceLimits(k.c); err != nil { glog.Warningf("unable to adjust resource limits: %v", err) } return nil @@ -560,7 +405,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { } files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) - if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { + if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { return errors.Wrap(err, "adding addons") } for _, f := range files { diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index e295fe170c84..eb75634792e8 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -52,7 +52,6 @@ import ( "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/registry" - pkgutil "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) @@ -343,7 +342,7 @@ func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) { func engineOptions(config cfg.MachineConfig) *engine.Options { o := engine.Options{ Env: config.DockerEnv, - InsecureRegistry: append([]string{pkgutil.DefaultServiceCIDR}, config.InsecureRegistry...), + InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, config.InsecureRegistry...), RegistryMirror: config.RegistryMirror, ArbitraryFlags: config.DockerOpt, InstallURL: drivers.DefaultEngineInstallURL, diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 1c7a3b665037..8dbcf11e1317 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "os" "os/exec" + "path" "time" "github.com/golang/glog" @@ -159,9 +160,9 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error { if out, err := exec.Command(k.ociBin, "cp", assetName, destination).CombinedOutput(); err != nil { return errors.Wrapf(err, "error copying %s into node, output: %s", f.GetAssetName(), string(out)) } - - if _, err := k.RunCmd(exec.Command("chmod", f.GetPermissions(), f.GetTargetDir())); err != nil { - return errors.Wrap(err, "failed to chmod file permissions") + fp := path.Join(f.GetTargetDir(), f.GetTargetName()) + if _, err := k.RunCmd(exec.Command("sudo", "chmod", f.GetPermissions(), fp)); err != nil { + return errors.Wrapf(err, "failed to chmod file permissions %s", fp) } return nil } diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 79e7f186918c..18bd9a75b4f3 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -81,8 +81,8 @@ type KubernetesConfig struct { ContainerRuntime string CRISocket string NetworkPlugin string - FeatureGates string - ServiceCIDR string + FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + ServiceCIDR string // the subnet which kubernetes services will be deployed to ImageRepository string ExtraOptions ExtraOptionSlice diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 9d328de7f3d2..85f6d40a7789 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -33,6 +33,8 @@ const ( APIServerName = "minikubeCA" // ClusterDNSDomain is the default DNS domain ClusterDNSDomain = "cluster.local" + // DefaultServiceCIDR is The CIDR to be used for service cluster IPs + DefaultServiceCIDR = "10.96.0.0/12" ) // DefaultMinipath is the default Minikube path (under the home directory) diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 1aea66e46621..31118c06391e 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/golang/glog" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/registry" ) @@ -81,7 +82,7 @@ func BareMetal(name string) bool { // FlagHints are hints for what default options should be used for this driver type FlagHints struct { - ExtraOptions string + ExtraOptions []string CacheImages bool ContainerRuntime string Bootstrapper string @@ -89,24 +90,26 @@ type FlagHints struct { // FlagDefaults returns suggested defaults based on a driver func FlagDefaults(name string) FlagHints { + fh := FlagHints{} if name != None { - fh := FlagHints{CacheImages: true} + fh.CacheImages = true // only for kic, till other run-times are available we auto-set containerd. if name == Docker { fh.ContainerRuntime = "containerd" fh.Bootstrapper = bootstrapper.KIC + fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) } return fh } - extraOpts := "" + fh.CacheImages = false + // if specifc linux add this option for systemd work on none driver if _, err := os.Stat(systemdResolvConf); err == nil { - extraOpts = fmt.Sprintf("kubelet.resolv-conf=%s", systemdResolvConf) - } - return FlagHints{ - ExtraOptions: extraOpts, - CacheImages: false, + noneEO := fmt.Sprintf("kubelet.resolv-conf=%s", systemdResolvConf) + fh.ExtraOptions = append(fh.ExtraOptions, noneEO) + return fh } + return fh } // Choices returns a list of drivers which are possible on this system diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index 7f07f9d62e8f..5aad1274b42d 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -72,7 +72,7 @@ func TestFlagDefaults(t *testing.T) { expected = FlagHints{ CacheImages: false, - ExtraOptions: fmt.Sprintf("kubelet.resolv-conf=%s", tf.Name()), + ExtraOptions: []string{fmt.Sprintf("kubelet.resolv-conf=%s", tf.Name())}, } systemdResolvConf = tf.Name() if diff := cmp.Diff(FlagDefaults(None), expected); diff != "" { diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 9abfc1d54627..42fc292b8711 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -21,8 +21,8 @@ import ( "os/exec" "github.com/docker/machine/libmachine/drivers" - "github.com/golang/glog" "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" @@ -33,7 +33,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Docker, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Docker}) }, Status: status, Priority: registry.Discouraged, // experimental }); err != nil { @@ -42,18 +42,14 @@ func init() { } func configure(mc config.MachineConfig) interface{} { - img, err := kic.ImageForVersion(mc.KubernetesConfig.KubernetesVersion) - if err != nil { - glog.Errorf("err to getting kic image for %s: imgesha:%s", img, mc.KubernetesConfig.KubernetesVersion) - } return kic.NewDriver(kic.Config{ - MachineName: mc.Name, - StorePath: localpath.MiniPath(), - ImageDigest: img, - CPU: mc.CPUs, - Memory: mc.Memory, - APIServerPort: mc.NodeBindPort, - OCIBinary: "docker", + MachineName: mc.Name, + StorePath: localpath.MiniPath(), + ImageDigest: kic.BaseImage, + CPU: mc.CPUs, + Memory: mc.Memory, + HostBindPort: mc.KubernetesConfig.NodePort, + OCIBinary: oci.Docker, }) } diff --git a/pkg/util/constants.go b/pkg/util/constants.go index f1132155c4e2..5020b7931330 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -24,9 +24,8 @@ import ( // These constants are used by both minikube const ( - APIServerPort = 8443 - DefaultDNSDomain = "cluster.local" - DefaultServiceCIDR = "10.96.0.0/12" + APIServerPort = 8443 + DefaultDNSDomain = "cluster.local" ) // DefaultV114AdmissionControllers are admission controllers we default to in v1.14.x diff --git a/site/.gitignore b/site/.gitignore new file mode 100644 index 000000000000..b512c09d4766 --- /dev/null +++ b/site/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file