auto-pause addon: automatically pause Kubernetes when not in use

.gitignore

@@ -26,6 +26,7 @@ _testmain.go
 *.test
 *.prof
 
+/deploy/kicbase/auto-pause
 /out
 /_gopath
 

Makefile

@@ -648,7 +648,7 @@ KICBASE_ARCH = linux/arm64,linux/amd64
 KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
 .PHONY: push-kic-base-image 
-push-kic-base-image: docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
+push-kic-base-image: deploy/kicbase/auto-pause docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube
 	docker login docker.pkg.github.com
@@ -812,6 +812,10 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo ## Serv
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/mkcmp/main.go
 
+.PHONY: deploy/kicbase/auto-pause # auto pause binary to be used for kic image work arround for not passing the whole repo as docker context
+deploy/kicbase/auto-pause: $(SOURCE_GENERATED) $(SOURCE_FILES)
+	GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
+
 .PHONY: out/performance-bot
 out/performance-bot:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go

cmd/auto-pause/auto-pause.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/reason"
+	"k8s.io/minikube/pkg/minikube/style"
+)
+
+var unpauseRequests = make(chan struct{})
+var done = make(chan struct{})
+var mu sync.Mutex
+
+// TODO: initialize with current state (handle the case that user enables auto-pause after it is already paused)
+var runtimePaused = false
+var version = "0.0.1"
+
+// TODO: #10597 make this configurable to support containerd/cri-o
+var runtime = "docker"
+
+func main() {
+	// TODO: #10595 make this configurable
+	const interval = time.Minute * 1
+	// channel for incoming messages
+	go func() {
+		for {
+			// On each iteration new timer is created
+			select {
+			// TODO: #10596 make it memory-leak proof
+			case <-time.After(interval):
+				runPause()
+			case <-unpauseRequests:
+				fmt.Printf("Got request\n")
+				if runtimePaused {
+					runUnpause()
+				}
+
+				done <- struct{}{}
+			}
+		}
+	}()
+
+	http.HandleFunc("/", handler) // each request calls handler
+	fmt.Printf("Starting auto-pause server %s at port 8080 \n", version)
+	log.Fatal(http.ListenAndServe("0.0.0.0:8080", nil))
+}
+
+// handler echoes the Path component of the requested URL.
+func handler(w http.ResponseWriter, r *http.Request) {
+	unpauseRequests <- struct{}{}
+	<-done
+	fmt.Fprintf(w, "allow")
+}
+
+func runPause() {
+	mu.Lock()
+	defer mu.Unlock()
+	if runtimePaused {
+		return
+	}
+
+	r := command.NewExecRunner(true)
+
+	cr, err := cruntime.New(cruntime.Config{Type: runtime, Runner: r})
+	if err != nil {
+		exit.Error(reason.InternalNewRuntime, "Failed runtime", err)
+	}
+
+	uids, err := cluster.Pause(cr, r, []string{"kube-system"})
+	if err != nil {
+		exit.Error(reason.GuestPause, "Pause", err)
+	}
+
+	runtimePaused = true
+
+	out.Step(style.Unpause, "Paused {{.count}} containers", out.V{"count": len(uids)})
+}
+
+func runUnpause() {
+	fmt.Println("unpausing...")
+	mu.Lock()
+	defer mu.Unlock()
+
+	r := command.NewExecRunner(true)
+
+	cr, err := cruntime.New(cruntime.Config{Type: runtime, Runner: r})
+	if err != nil {
+		exit.Error(reason.InternalNewRuntime, "Failed runtime", err)
+	}
+
+	uids, err := cluster.Unpause(cr, r, nil)
+	if err != nil {
+		exit.Error(reason.GuestUnpause, "Unpause", err)
+	}
+	runtimePaused = false
+
+	out.Step(style.Unpause, "Unpaused {{.count}} containers", out.V{"count": len(uids)})
+}

cmd/minikube/cmd/status.go

@@ -379,7 +379,14 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
 		return st, nil
 	}
 
-	hostname, _, port, err := driver.ControlPlaneEndpoint(&cc, &n, host.DriverName)
+	var hostname string
+	var port int
+	if cc.Addons["auto-pause"] {
+		hostname, _, port, err = driver.AutoPauseProxyEndpoint(&cc, &n, host.DriverName)
+	} else {
+		hostname, _, port, err = driver.ControlPlaneEndpoint(&cc, &n, host.DriverName)
+	}
+
 	if err != nil {
 		klog.Errorf("forwarded endpoint: %v", err)
 		st.Kubeconfig = Misconfigured

deploy/addons/auto-pause/auto-pause.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Auto Pause Service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/auto-pause
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

deploy/addons/auto-pause/auto-pause.yaml.tmpl

@@ -0,0 +1,47 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: auto-pause
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: auto-pause-proxy
+  namespace: auto-pause
+  labels:
+    app: auto-pause-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: auto-pause-proxy
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: auto-pause-proxy
+    spec:
+      volumes:
+      - name: ha-cfg
+        hostPath:
+          path: /var/lib/minikube/haproxy.cfg
+          type: File
+      - name: lua-script
+        hostPath:
+          path: /var/lib/minikube/unpause.lua
+          type: File
+      containers:
+        - name: auto-pause
+          image: "haproxy:2.3.5-alpine"
+          ports:
+            - name: https
+              containerPort: 6443
+              hostPort: 32443
+              protocol: TCP
+          volumeMounts:
+          - name: ha-cfg
+            mountPath: /usr/local/etc/haproxy/haproxy.cfg
+            readOnly: true
+          - name: lua-script
+            mountPath: /etc/haproxy/unpause.lua

deploy/addons/auto-pause/haproxy.cfg

@@ -0,0 +1,37 @@
+#---------------------------------------------------------------------
+# Configure HAProxy for Kubernetes API Server
+#---------------------------------------------------------------------
+listen stats
+  bind    *:9000
+  mode    http
+  stats   enable
+  stats   hide-version
+  stats   uri       /
+  stats   refresh   30s
+  option  httplog
+
+# change haproxy.cfg file with the following
+global
+   lua-load /etc/haproxy/unpause.lua
+
+############## Configure HAProxy Secure Frontend #############
+frontend k8s-api-https-proxy
+    bind *:6443
+    mode tcp
+    tcp-request inspect-delay 5s
+    tcp-request content accept if { req.ssl_hello_type 1 }
+    default_backend k8s-api-https
+############## Configure HAProxy SecureBackend #############
+backend k8s-api-https
+    balance roundrobin
+    mode tcp
+    #tcp-request inspect-delay 10s
+    #tcp-request content lua.foo_action
+    tcp-request inspect-delay 10s
+    tcp-request content lua.unpause 192.168.49.2 8080
+    tcp-request content reject if { var(req.blocked) -m bool }
+    option tcplog
+    option tcp-check
+    default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
+    server k8s-api-1 192.168.49.2:8443 check
+

deploy/addons/auto-pause/unpause.lua

@@ -0,0 +1,57 @@
+local function unpause(txn, addr, port)
+    if not addr then addr = '127.0.0.1' end
+    if not port then port = 5000 end
+
+    -- Set up a request to the service
+    local hdrs = {
+        [1] = string.format('host: %s:%s', addr, port),
+        [2] = 'accept: */*',
+        [3] = 'connection: close'
+    }
+
+    local req = {
+        [1] = string.format('GET /%s HTTP/1.1', tostring(txn.f:src())),
+        [2] = table.concat(hdrs, '\r\n'),
+        [3] = '\r\n'
+    }
+
+    req = table.concat(req,  '\r\n')
+
+    -- Use core.tcp to get an instance of the Socket class
+    local socket = core.tcp()
+    socket:settimeout(5)
+
+    -- Connect to the service and send the request
+    if socket:connect(addr, port) then
+        if socket:send(req) then
+            -- Skip response headers
+            while true do
+                local line, _ = socket:receive('*l')
+
+                if not line then break end
+                if line == '' then break end
+            end
+
+            -- Get response body, if any
+            local content = socket:receive('*a')
+
+            -- Check if this request should be allowed
+            if content and content == 'allow' then
+                txn:set_var('req.blocked', false)
+                return
+            end
+        else
+            core.Alert('Could not connect to IP Checker server (send)')
+        end
+
+        socket:close()
+    else
+        core.Alert('Could not connect to IP Checker server (connect)')
+    end
+
+    -- The request should be blocked
+    txn:set_var('req.blocked', true)
+end
+
+core.register_action('unpause', {'tcp-req'}, unpause, 2)
+

deploy/kicbase/Dockerfile

@@ -28,6 +28,8 @@ COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf
 COPY 11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf
 COPY clean-install /usr/local/bin/clean-install
 COPY entrypoint /usr/local/bin/entrypoint
+# must first run make out/auto-pause
+COPY auto-pause /usr/local/bin/auto-pause
 
 # Install dependencies, first from apt, then from release tarballs.
 # NOTE: we use one RUN to minimize layers.

pkg/addons/addons.go

@@ -38,12 +38,15 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/kubeconfig"
 	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/mustload"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/storageclass"
 	"k8s.io/minikube/pkg/minikube/style"
+	"k8s.io/minikube/pkg/minikube/sysinit"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -201,13 +204,19 @@ https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Dri
 		}
 	}
 
-	cmd, err := machine.CommandRunner(host)
+	runner, err := machine.CommandRunner(host)
 	if err != nil {
 		return errors.Wrap(err, "command runner")
 	}
 
+	if name == "auto-pause" && !enable { // needs to be disabled before deleting the service file in the internal disable
+		if err := sysinit.New(runner).DisableNow("auto-pause"); err != nil {
+			klog.ErrorS(err, "failed to disable", "service", "auto-pause")
+		}
+	}
+
 	data := assets.GenerateTemplateData(addon, cc.KubernetesConfig)
-	return enableOrDisableAddonInternal(cc, addon, cmd, data, enable)
+	return enableOrDisableAddonInternal(cc, addon, runner, data, enable)
 }
 
 func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable bool) bool {
@@ -223,7 +232,7 @@ func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable boo
 	return false
 }
 
-func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
+func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon, runner command.Runner, data interface{}, enable bool) error {
 	deployFiles := []string{}
 
 	for _, addon := range addon.Assets {
@@ -242,13 +251,13 @@ func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon,
 
 		if enable {
 			klog.Infof("installing %s", fPath)
-			if err := cmd.Copy(f); err != nil {
+			if err := runner.Copy(f); err != nil {
 				return err
 			}
 		} else {
 			klog.Infof("Removing %+v", fPath)
 			defer func() {
-				if err := cmd.Remove(f); err != nil {
+				if err := runner.Remove(f); err != nil {
 					klog.Warningf("error removing %s; addon should still be disabled as expected", fPath)
 				}
 			}()
@@ -260,7 +269,7 @@ func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon,
 
 	// Retry, because sometimes we race against an apiserver restart
 	apply := func() error {
-		_, err := cmd.RunCmd(kubectlCommand(cc, deployFiles, enable))
+		_, err := runner.RunCmd(kubectlCommand(cc, deployFiles, enable))
 		if err != nil {
 			klog.Warningf("apply failed, will retry: %v", err)
 		}
@@ -435,3 +444,47 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo
 		}
 	}
 }
+
+// enableOrDisableAutoPause enables the service after the config was copied by generic enble
+func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) error {
+	enable, err := strconv.ParseBool(val)
+	if err != nil {
+		return errors.Wrapf(err, "parsing bool: %s", name)
+	}
+	out.Infof("auto-pause addon is an alpha feature and still in early development. Please file issues to help us make it better.")
+	out.Infof("https://github.com/kubernetes/minikube/labels/co%2Fauto-pause")
+
+	if !driver.IsKIC(cc.Driver) || runtime.GOARCH != "amd64" {
+		exit.Message(reason.Usage, `auto-pause currently is only supported on docker driver/docker runtime/amd64. Track progress of others here: https://github.com/kubernetes/minikube/issues/10601`)
+	}
+	co := mustload.Running(cc.Name)
+	if enable {
+		if err := sysinit.New(co.CP.Runner).EnableNow("auto-pause"); err != nil {
+			klog.ErrorS(err, "failed to enable", "service", "auto-pause")
+		}
+	}
+
+	port := co.CP.Port // api server port
+	if enable {        // if enable then need to calculate the forwarded port
+		port = constants.AutoPauseProxyPort
+		if driver.NeedsPortForward(cc.Driver) {
+			port, err = oci.ForwardedPort(cc.Driver, cc.Name, port)
+			if err != nil {
+				klog.ErrorS(err, "failed to get forwarded port for", "auto-pause port", port)
+			}
+		}
+	}
+
+	updated, err := kubeconfig.UpdateEndpoint(cc.Name, co.CP.Hostname, port, kubeconfig.PathFromEnv(), kubeconfig.NewExtension())
+	if err != nil {
+		klog.ErrorS(err, "failed to update kubeconfig", "auto-pause proxy endpoint")
+		return err
+	}
+	if updated {
+		klog.Infof("%s context has been updated to point to auto-pause proxy %s:%s", cc.Name, co.CP.Hostname, co.CP.Port)
+	} else {
+		klog.Info("no need to update kube-context for auto-pause proxy")
+	}
+
+	return nil
+}

pkg/addons/config.go

@@ -42,6 +42,12 @@ var addonPodLabels = map[string]string{
 
 // Addons is a list of all addons
 var Addons = []*Addon{
+	{
+		name:      "auto-pause",
+		set:       SetBool,
+		callbacks: []setFn{EnableOrDisableAddon, enableOrDisableAutoPause},
+	},
+
 	{
 		name:      "dashboard",
 		set:       SetBool,

pkg/drivers/kic/kic.go

@@ -127,6 +127,10 @@ func (d *Driver) Create() error {
 			ListenAddress: listAddr,
 			ContainerPort: constants.RegistryAddonPort,
 		},
+		oci.PortMapping{
+			ListenAddress: listAddr,
+			ContainerPort: constants.AutoPauseProxyPort,
+		},
 	)
 
 	exists, err := oci.ContainerExists(d.OCIBinary, params.Name, true)

pkg/drivers/kic/types.go

@@ -24,9 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.17-1613934488-10548"
+	Version = "v0.0.17-1614202509-10427"
 	// SHA of the kic base image
-	baseImageSHA = "5cacd48d07f699a171eedf65ef1490bd59a523ffcd90662e3b66eb838c5a1b5d"
+	baseImageSHA = "93f2448d272ebad3d564c04474cafe03bb7440149c241d1d010b2e90e14da213"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository

pkg/minikube/assets/addons.go

@@ -71,6 +71,34 @@ func (a *Addon) IsEnabled(cc *config.ClusterConfig) bool {
 // Addons is the list of addons
 // TODO: Make dynamically loadable: move this data to a .yaml file within each addon directory
 var Addons = map[string]*Addon{
+	"auto-pause": NewAddon([]*BinAsset{
+		MustBinAsset(
+			"deploy/addons/auto-pause/auto-pause.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"auto-pause.yaml",
+			"0640"),
+		MustBinAsset(
+			"deploy/addons/auto-pause/haproxy.cfg",
+			"/var/lib/minikube/",
+			"haproxy.cfg",
+			"0640"),
+		MustBinAsset(
+			"deploy/addons/auto-pause/unpause.lua",
+			"/var/lib/minikube/",
+			"unpause.lua",
+			"0640"),
+		MustBinAsset(
+			"deploy/addons/auto-pause/auto-pause.service",
+			"/etc/systemd/system/",
+			"auto-pause.service",
+			"0640"),
+
+		//GuestPersistentDir
+	}, false, "auto-pause", map[string]string{
+		"haproxy": "haproxy:2.3.5",
+	}, map[string]string{
+		"haproxy": "gcr.io",
+	}),
 	"dashboard": NewAddon([]*BinAsset{
 		// We want to create the kubernetes-dashboard ns first so that every subsequent object can be created
 		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640"),

pkg/minikube/constants/constants.go

@@ -41,6 +41,9 @@ const (
 	DockerDaemonPort = 2376
 	// APIServerPort is the default API server port
 	APIServerPort = 8443
+	// AutoPauseProxyPort is the port to be used as a reverse proxy for apiserver port
+	AutoPauseProxyPort = 32443
+
 	// SSHPort is the SSH serviceport on the node vm and container
 	SSHPort = 22
 	// RegistryAddonPort os the default registry addon port

pkg/minikube/driver/endpoint.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 
+	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -29,6 +30,9 @@ import (
 func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) {
 	if NeedsPortForward(driverName) {
 		port, err := oci.ForwardedPort(cc.Driver, cc.Name, cp.Port)
+		if err != nil {
+			klog.Warningf("failed to get forwarded control plane port %v", err)
+		}
 		hostname := oci.DaemonHost(driverName)
 
 		ip := net.ParseIP(hostname)
@@ -54,3 +58,9 @@ func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName
 	}
 	return hostname, ip, cp.Port, nil
 }
+
+// AutoPauseProxyEndpoint returns the endpoint for the auto-pause (reverse proxy to api-sever)
+func AutoPauseProxyEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) {
+	cp.Port = constants.AutoPauseProxyPort
+	return ControlPlaneEndpoint(cc, cp, driverName)
+}

pkg/minikube/sysinit/openrc.go

@@ -117,11 +117,21 @@ func (s *OpenRC) Disable(svc string) error {
 	return nil
 }
 
+// DisableNow not implemented for openRC
+func (s *OpenRC) DisableNow(svc string) error {
+	return fmt.Errorf("disable now is not implemented for OpenRC! PRs to fix are welcomed")
+}
+
 // Enable does nothing
 func (s *OpenRC) Enable(svc string) error {
 	return nil
 }
 
+// EnableNow  not implemented for openRC
+func (s *OpenRC) EnableNow(svc string) error {
+	return fmt.Errorf("enable now is not implemented for OpenRC! PRs to fix are welcomed")
+}
+
 // Restart restarts a service
 func (s *OpenRC) Restart(svc string) error {
 	rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "restart"))

pkg/minikube/sysinit/sysinit.go

@@ -41,9 +41,15 @@ type Manager interface {
 	// Disable disables a service
 	Disable(string) error
 
+	// Disable disables a service and stops it right after.
+	DisableNow(string) error
+
 	// Enable enables a service
 	Enable(string) error
 
+	// EnableNow enables a service and starts it right after.
+	EnableNow(string) error
+
 	// Start starts a service idempotently
 	Start(string) error
 

pkg/minikube/sysinit/systemd.go

@@ -52,6 +52,12 @@ func (s *Systemd) Disable(svc string) error {
 	return err
 }
 
+// DisableNow disables a service and stops it too (not waiting for next restart)
+func (s *Systemd) DisableNow(svc string) error {
+	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "disable", "--now", svc))
+	return err
+}
+
 // Enable enables a service
 func (s *Systemd) Enable(svc string) error {
 	if svc == "kubelet" {
@@ -61,6 +67,15 @@ func (s *Systemd) Enable(svc string) error {
 	return err
 }
 
+// Enable enables a service and then activates it too (not waiting for next start)
+func (s *Systemd) EnableNow(svc string) error {
+	if svc == "kubelet" {
+		return errors.New("please don't enable kubelet as it creates a race condition; if it starts on systemd boot it will pick up /etc/hosts before we have time to configure /etc/hosts")
+	}
+	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "enable", "--now", svc))
+	return err
+}
+
 // Start starts a service
 func (s *Systemd) Start(svc string) error {
 	if err := s.daemonReload(); err != nil {

site/content/en/docs/commands/start.md

@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613934488-10548@sha256:5cacd48d07f699a171eedf65ef1490bd59a523ffcd90662e3b66eb838c5a1b5d")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1614202509-10427@sha256:93f2448d272ebad3d564c04474cafe03bb7440149c241d1d010b2e90e14da213")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")