Fix none driver bugs with "pause"

cmd/minikube/cmd/delete.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -194,7 +195,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	}
 
 	if err == nil && driver.BareMetal(cc.VMDriver) {
-		if err := uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
+		if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
 			deletionError, ok := err.(DeletionError)
 			if ok {
 				delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
@@ -276,12 +277,34 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
 	return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
 }
 
-func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) error {
+func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error {
 	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
 	clusterBootstrapper, err := getClusterBootstrapper(api, bsName)
 	if err != nil {
 		return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
-	} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r})
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	// Unpause the cluster if necessary to avoid hung kubeadm
+	_, err = cluster.Unpause(cr, r, nil)
+	if err != nil {
+		glog.Errorf("unpause failed: %v", err)
+	}
+
+	if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
 		return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal}
 	}
 	return nil

cmd/minikube/cmd/status.go

@@ -30,7 +30,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -142,24 +142,31 @@ func status(api libmachine.API, name string) (*Status, error) {
 		return st, errors.Wrap(err, "host")
 	}
 
-	// Nonexistent it is!
 	if hs == state.None.String() {
 		return st, nil
 	}
+
 	st.Host = hs
-	if st.Host != state.Running.String() {
+	if st.Host != state.Running.String() && st.Host != state.Paused.String() {
 		return st, nil
 	}
 
-	bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
 	if err != nil {
-		return st, errors.Wrap(err, "bootstrapper")
+		return st, err
+	}
+
+	cr, err := machine.CommandRunner(host)
+	if err != nil {
+		return st, err
 	}
 
-	st.Kubelet, err = bs.GetKubeletStatus()
+	stk, err := kverify.KubeletStatus(cr)
 	if err != nil {
 		glog.Warningf("kubelet err: %v", err)
 		st.Kubelet = state.Error.String()
+	} else {
+		st.Kubelet = stk.String()
 	}
 
 	ip, err := cluster.GetHostDriverIP(api, name)
@@ -175,10 +182,12 @@ func status(api libmachine.API, name string) (*Status, error) {
 		port = constants.APIServerPort
 	}
 
-	st.APIServer, err = bs.GetAPIServerStatus(ip, port)
+	sta, err := kverify.APIServerStatus(cr, ip, port)
 	if err != nil {
 		glog.Errorln("Error apiserver status:", err)
 		st.APIServer = state.Error.String()
+	} else {
+		st.APIServer = sta.String()
 	}
 
 	st.Kubeconfig = Misconfigured

pkg/drivers/none/none.go

@@ -26,7 +26,7 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"k8s.io/apimachinery/pkg/util/net"
+	knet "k8s.io/apimachinery/pkg/util/net"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -94,7 +94,7 @@ func (d *Driver) DriverName() string {
 
 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	ip, err := net.ChooseHostInterface()
+	ip, err := knet.ChooseHostInterface()
 	if err != nil {
 		return "", err
 	}
@@ -123,10 +123,13 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	if err := checkKubelet(d.exec); err != nil {
-		glog.Infof("kubelet not running: %v", err)
-		return state.Stopped, nil
+	_, err := d.GetIP()
+	if err != nil {
+		return state.Error, err
 	}
+
+	// If we got this far, it's safe to call the host as running like a VM would.
+	// Rely on other checks to determine if kubelet and apiserver are healthy.
 	return state.Running, nil
 }
 
@@ -251,13 +254,3 @@ func restartKubelet(cr command.Runner) error {
 	}
 	return nil
 }
-
-// checkKubelet returns an error if the kubelet is not running.
-func checkKubelet(cr command.Runner) error {
-	glog.Infof("checking for running kubelet ...")
-	c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "check kubelet")
-	}
-	return nil
-}

pkg/minikube/bootstrapper/bsutil/kverify/kverify.go

@@ -23,6 +23,8 @@ import (
 	"net"
 	"net/http"
 	"os/exec"
+	"path"
+	"strings"
 	"time"
 
 	"github.com/docker/machine/libmachine/state"
@@ -101,12 +103,12 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
 			return false, fmt.Errorf("cluster wait timed out during healthz check")
 		}
 
-		status, err := APIServerStatus(net.ParseIP(ip), port)
+		status, err := apiServerHealthz(net.ParseIP(ip), port)
 		if err != nil {
 			glog.Warningf("status: %v", err)
 			return false, nil
 		}
-		if status != "Running" {
+		if status != state.Running {
 			return false, nil
 		}
 		return true, nil
@@ -119,9 +121,53 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
 	return nil
 }
 
-// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State
-func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
-	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort)))
+// APIServerStatus returns apiserver status in libmachine style state.State
+func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
+	glog.Infof("Checking apiserver status ...")
+	// sudo, in case hidepid is set
+	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+	if err != nil {
+		return state.Stopped, nil
+	}
+	pids := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n")
+	pid := pids[len(pids)-1]
+	if len(pids) != 1 {
+		glog.Errorf("found %d apiserver pids: %v - choosing %s", len(pids), pids, pid)
+	}
+
+	// Get the freezer cgroup entry for this pid
+	rr, err = cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
+	if err != nil {
+		glog.Warningf("unable to find freezer cgroup: %v", err)
+		return apiServerHealthz(ip, port)
+
+	}
+	freezer := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("apiserver freezer: %q", freezer)
+	fparts := strings.Split(freezer, ":")
+	if len(fparts) != 3 {
+		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
+		return apiServerHealthz(ip, port)
+	}
+
+	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
+	if err != nil {
+		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
+		return apiServerHealthz(ip, port)
+	}
+
+	fs := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("freezer state: %q", fs)
+	if fs == "FREEZING" || fs == "FROZEN" {
+		return state.Paused, nil
+	}
+	return apiServerHealthz(ip, port)
+}
+
+// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
+func apiServerHealthz(ip net.IP, port int) (state.State, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
+	glog.Infof("Checking apiserver healthz at %s ...", url)
 	// To avoid: x509: certificate signed by unknown authority
 	tr := &http.Transport{
 		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
@@ -131,11 +177,31 @@ func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
 	resp, err := client.Get(url)
 	// Connection refused, usually.
 	if err != nil {
-		return state.Stopped.String(), nil
+		return state.Stopped, nil
 	}
 	if resp.StatusCode != http.StatusOK {
 		glog.Warningf("%s response: %v %+v", url, err, resp)
-		return state.Error.String(), nil
+		return state.Error, nil
+	}
+	return state.Running, nil
+}
+
+func KubeletStatus(cr command.Runner) (state.State, error) {
+	glog.Infof("Checking kubelet status ...")
+	rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
+	if err != nil {
+		// Do not return now, as we still have parsing to do!
+		glog.Warningf("%s returned error: %v", rr.Command(), err)
+	}
+	s := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("kubelet is-active: %s", s)
+	switch s {
+	case "active":
+		return state.Running, nil
+	case "inactive":
+		return state.Stopped, nil
+	case "activating":
+		return state.Starting, nil
 	}
-	return state.Running.String(), nil
+	return state.Error, nil
 }

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -98,42 +98,12 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) {
 }
 
 // GetAPIServerStatus returns the api-server status
-func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) {
-	// sudo, in case hidepid is set
-	rr, err := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, port int) (string, error) {
+	s, err := kverify.APIServerStatus(k.c, ip, port)
 	if err != nil {
-		return state.Stopped.String(), nil
-	}
-	pid := strings.TrimSpace(rr.Stdout.String())
-
-	// Get the freezer cgroup entry for this pid
-	rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
-	if err != nil {
-		glog.Warningf("unable to find freezer cgroup: %v", err)
-		return kverify.APIServerStatus(ip, apiserverPort)
-
-	}
-	freezer := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("apiserver freezer: %q", freezer)
-	fparts := strings.Split(freezer, ":")
-	if len(fparts) != 3 {
-		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
-		return kverify.APIServerStatus(ip, apiserverPort)
+		return state.Error.String(), err
 	}
-
-	rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
-	if err != nil {
-		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
-		return kverify.APIServerStatus(ip, apiserverPort)
-	}
-
-	fs := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("freezer state: %q", fs)
-	if fs == "FREEZING" || fs == "FROZEN" {
-		return state.Paused.String(), nil
-	}
-
-	return kverify.APIServerStatus(ip, apiserverPort)
+	return s.String(), nil
 }
 
 // LogCommands returns a map of log type to a command which will display that log.