Merge pull request #7389 from sharifelgamal/driver-fallback

fallback to alternate drivers on failure

Author: sharifelgamal

cmd/minikube/cmd/node_add.go

@@ -20,6 +20,7 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/mustload"
 	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -54,7 +55,10 @@ var nodeAddCmd = &cobra.Command{
 		}
 
 		if err := node.Add(cc, n); err != nil {
-			maybeDeleteAndRetry(*cc, n, nil, err)
+			_, err := maybeDeleteAndRetry(*cc, n, nil, err)
+			if err != nil {
+				exit.WithError("failed to add node", err)
+			}
 		}
 
 		out.T(out.Ready, "Successfully added {{.name}} to {{.cluster}}!", out.V{"name": name, "cluster": cc.Name})

cmd/minikube/cmd/node_start.go

@@ -49,9 +49,27 @@ var nodeStartCmd = &cobra.Command{
 			exit.WithError("retrieving node", err)
 		}
 
-		_, err = node.Start(*cc, *n, nil, false)
+		r, p, m, h, err := node.Provision(cc, n, false)
 		if err != nil {
-			maybeDeleteAndRetry(*cc, *n, nil, err)
+			exit.WithError("provisioning host for node", err)
+		}
+
+		s := node.Starter{
+			Runner:         r,
+			PreExists:      p,
+			MachineAPI:     m,
+			Host:           h,
+			Cfg:            cc,
+			Node:           n,
+			ExistingAddons: nil,
+		}
+
+		_, err = node.Start(s, false)
+		if err != nil {
+			_, err := maybeDeleteAndRetry(*cc, *n, nil, err)
+			if err != nil {
+				exit.WithError("failed to start node", err)
+			}
 		}
 	},
 }

cmd/minikube/cmd/start.go

@@ -150,11 +150,59 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	validateSpecifiedDriver(existing)
-	ds := selectDriver(existing)
+	ds, alts, specified := selectDriver(existing)
+	starter, err := provisionWithDriver(cmd, ds, existing)
+	if err != nil {
+		if specified {
+			// If the user specified a driver, don't fallback to anything else
+			exit.WithError("error provisioning host", err)
+		} else {
+			success := false
+			// Walk down the rest of the options
+			for _, alt := range alts {
+				out.WarningT("Startup with {{.old_driver}} driver failed, trying with alternate driver {{.new_driver}}: {{.error}}", out.V{"old_driver": ds.Name, "new_driver": alt.Name, "error": err})
+				ds = alt
+				// Delete the existing cluster and try again with the next driver on the list
+				profile, err := config.LoadProfile(ClusterFlagValue())
+				if err != nil {
+					glog.Warningf("%s profile does not exist, trying anyways.", ClusterFlagValue())
+				}
+
+				err = deleteProfile(profile)
+				if err != nil {
+					out.WarningT("Failed to delete cluster {{.name}}, proceeding with retry anyway.", out.V{"name": ClusterFlagValue()})
+				}
+				starter, err = provisionWithDriver(cmd, ds, existing)
+				if err != nil {
+					continue
+				} else {
+					// Success!
+					success = true
+					break
+				}
+			}
+			if !success {
+				exit.WithError("error provisioning host", err)
+			}
+		}
+	}
+
+	kubeconfig, err := startWithDriver(starter, existing)
+	if err != nil {
+		exit.WithError("failed to start node", err)
+	}
+
+	if err := showKubectlInfo(kubeconfig, starter.Node.KubernetesVersion, starter.Cfg.Name); err != nil {
+		glog.Errorf("kubectl info: %v", err)
+	}
+
+}
+
+func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *config.ClusterConfig) (node.Starter, error) {
 	driverName := ds.Name
 	glog.Infof("selected driver: %s", driverName)
 	validateDriver(ds, existing)
-	err = autoSetDriverOptions(cmd, driverName)
+	err := autoSetDriverOptions(cmd, driverName)
 	if err != nil {
 		glog.Errorf("Error autoSetOptions : %v", err)
 	}
@@ -170,19 +218,19 @@ func runStart(cmd *cobra.Command, args []string) {
 	k8sVersion := getKubernetesVersion(existing)
 	cc, n, err := generateClusterConfig(cmd, existing, k8sVersion, driverName)
 	if err != nil {
-		exit.WithError("Failed to generate config", err)
+		return node.Starter{}, errors.Wrap(err, "Failed to generate config")
 	}
 
 	// This is about as far as we can go without overwriting config files
 	if viper.GetBool(dryRun) {
 		out.T(out.DryRun, `dry-run validation complete!`)
-		return
+		os.Exit(0)
 	}
 
 	if driver.IsVM(driverName) {
 		url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL))
 		if err != nil {
-			exit.WithError("Failed to cache ISO", err)
+			return node.Starter{}, errors.Wrap(err, "Failed to cache ISO")
 		}
 		cc.MinikubeISO = url
 	}
@@ -201,17 +249,37 @@ func runStart(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	kubeconfig, err := node.Start(cc, n, existingAddons, true)
+	mRunner, preExists, mAPI, host, err := node.Provision(&cc, &n, true)
 	if err != nil {
-		kubeconfig = maybeDeleteAndRetry(cc, n, existingAddons, err)
+		return node.Starter{}, err
+	}
+
+	return node.Starter{
+		Runner:         mRunner,
+		PreExists:      preExists,
+		MachineAPI:     mAPI,
+		Host:           host,
+		ExistingAddons: existingAddons,
+		Cfg:            &cc,
+		Node:           &n,
+	}, nil
+}
+
+func startWithDriver(starter node.Starter, existing *config.ClusterConfig) (*kubeconfig.Settings, error) {
+	kubeconfig, err := node.Start(starter, true)
+	if err != nil {
+		kubeconfig, err = maybeDeleteAndRetry(*starter.Cfg, *starter.Node, starter.ExistingAddons, err)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	numNodes := viper.GetInt(nodes)
 	if numNodes == 1 && existing != nil {
 		numNodes = len(existing.Nodes)
 	}
 	if numNodes > 1 {
-		if driver.BareMetal(driverName) {
+		if driver.BareMetal(starter.Cfg.Driver) {
 			exit.WithCodeT(exit.Config, "The none driver is not compatible with multi-node clusters.")
 		} else {
 			for i := 1; i < numNodes; i++ {
@@ -220,20 +288,18 @@ func runStart(cmd *cobra.Command, args []string) {
 					Name:              nodeName,
 					Worker:            true,
 					ControlPlane:      false,
-					KubernetesVersion: cc.KubernetesConfig.KubernetesVersion,
+					KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion,
 				}
 				out.Ln("") // extra newline for clarity on the command line
-				err := node.Add(&cc, n)
+				err := node.Add(starter.Cfg, n)
 				if err != nil {
-					exit.WithError("adding node", err)
+					return nil, errors.Wrap(err, "adding node")
 				}
 			}
 		}
 	}
 
-	if err := showKubectlInfo(kubeconfig, cc.KubernetesConfig.KubernetesVersion, cc.Name); err != nil {
-		glog.Errorf("kubectl info: %v", err)
-	}
+	return kubeconfig, nil
 }
 
 func updateDriver(driverName string) {
@@ -303,7 +369,7 @@ func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName st
 	return nil
 }
 
-func maybeDeleteAndRetry(cc config.ClusterConfig, n config.Node, existingAddons map[string]bool, originalErr error) *kubeconfig.Settings {
+func maybeDeleteAndRetry(cc config.ClusterConfig, n config.Node, existingAddons map[string]bool, originalErr error) (*kubeconfig.Settings, error) {
 	if viper.GetBool(deleteOnFailure) {
 		out.WarningT("Node {{.name}} failed to start, deleting and trying again.", out.V{"name": n.Name})
 		// Start failed, delete the cluster and try again
@@ -318,21 +384,35 @@ func maybeDeleteAndRetry(cc config.ClusterConfig, n config.Node, existingAddons
 		}
 
 		var kubeconfig *kubeconfig.Settings
-		for _, v := range cc.Nodes {
-			k, err := node.Start(cc, v, existingAddons, v.ControlPlane)
-			if v.ControlPlane {
+		for _, n := range cc.Nodes {
+			r, p, m, h, err := node.Provision(&cc, &n, n.ControlPlane)
+			s := node.Starter{
+				Runner:         r,
+				PreExists:      p,
+				MachineAPI:     m,
+				Host:           h,
+				Cfg:            &cc,
+				Node:           &n,
+				ExistingAddons: existingAddons,
+			}
+			if err != nil {
+				// Ok we failed again, let's bail
+				return nil, err
+			}
+
+			k, err := node.Start(s, n.ControlPlane)
+			if n.ControlPlane {
 				kubeconfig = k
 			}
 			if err != nil {
 				// Ok we failed again, let's bail
-				exit.WithError("Start failed after cluster deletion", err)
+				return nil, err
 			}
 		}
-		return kubeconfig
+		return kubeconfig, nil
 	}
 	// Don't delete the cluster unless they ask
-	exit.WithError("startup failed", originalErr)
-	return nil
+	return nil, errors.Wrap(originalErr, "startup failed")
 }
 
 func kubectlVersion(path string) (string, error) {
@@ -360,7 +440,7 @@ func kubectlVersion(path string) (string, error) {
 	return cv.ClientVersion.GitVersion, nil
 }
 
-func selectDriver(existing *config.ClusterConfig) registry.DriverState {
+func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []registry.DriverState, bool) {
 	// Technically unrelated, but important to perform before detection
 	driver.SetLibvirtURI(viper.GetString(kvmQemuURI))
 
@@ -369,7 +449,7 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 		old := hostDriver(existing)
 		ds := driver.Status(old)
 		out.T(out.Sparkle, `Using the {{.driver}} driver based on existing profile`, out.V{"driver": ds.String()})
-		return ds
+		return ds, nil, true
 	}
 
 	// Default to looking at the new driver parameter
@@ -389,7 +469,7 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 			exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": d, "os": runtime.GOOS})
 		}
 		out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
-		return ds
+		return ds, nil, true
 	}
 
 	// Fallback to old driver parameter
@@ -399,7 +479,7 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 			exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": d, "os": runtime.GOOS})
 		}
 		out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
-		return ds
+		return ds, nil, true
 	}
 
 	choices := driver.Choices(viper.GetBool("vm"))
@@ -422,7 +502,7 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 	} else {
 		out.T(out.Sparkle, `Automatically selected the {{.driver}} driver`, out.V{"driver": pick.String()})
 	}
-	return pick
+	return pick, alts, false
 }
 
 // hostDriver returns the actual driver used by a libmachine host, which can differ from our config

pkg/minikube/exit/exit.go

@@ -18,15 +18,13 @@ limitations under the License.
 package exit
 
 import (
-	"fmt"
 	"os"
 	"runtime"
 	"runtime/debug"
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/problem"
-	"k8s.io/minikube/pkg/minikube/translate"
 )
 
 // Exit codes based on sysexits(3)
@@ -41,9 +39,6 @@ const (
 	IO          = 74 // IO represents an I/O error
 	Config      = 78 // Config represents an unconfigured or misconfigured state
 	Permissions = 77 // Permissions represents a permissions error
-
-	// MaxLogEntries controls the number of log entries to show for each source
-	MaxLogEntries = 3
 )
 
 // UsageT outputs a templated usage error and exits with error code 64
@@ -65,7 +60,7 @@ func WithError(msg string, err error) {
 	if p != nil {
 		WithProblem(msg, err, p)
 	}
-	displayError(msg, err)
+	out.DisplayError(msg, err)
 	os.Exit(Software)
 }
 
@@ -81,29 +76,3 @@ func WithProblem(msg string, err error, p *problem.Problem) {
 	}
 	os.Exit(Config)
 }
-
-// WithLogEntries outputs an error along with any important log entries, and exits.
-func WithLogEntries(msg string, err error, entries map[string][]string) {
-	displayError(msg, err)
-
-	for name, lines := range entries {
-		out.FailureT("Problems detected in {{.entry}}:", out.V{"entry": name})
-		if len(lines) > MaxLogEntries {
-			lines = lines[:MaxLogEntries]
-		}
-		for _, l := range lines {
-			out.T(out.LogEntry, l)
-		}
-	}
-	os.Exit(Software)
-}
-
-func displayError(msg string, err error) {
-	// use Warning because Error will display a duplicate message to stderr
-	glog.Warningf(fmt.Sprintf("%s: %v", msg, err))
-	out.ErrT(out.Empty, "")
-	out.FatalT("{{.msg}}: {{.err}}", out.V{"msg": translate.T(msg), "err": err})
-	out.ErrT(out.Empty, "")
-	out.ErrT(out.Sad, "minikube is exiting due to an error. If the above message is not useful, open an issue:")
-	out.ErrT(out.URL, "https://github.com/kubernetes/minikube/issues/new/choose")
-}

pkg/minikube/node/cache.go

@@ -80,7 +80,6 @@ func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion string)
 	}
 	out.T(out.Check, "Download complete!")
 	os.Exit(0)
-
 }
 
 // CacheKubectlBinary caches the kubectl binary

pkg/minikube/node/node.go

@@ -39,7 +39,21 @@ func Add(cc *config.ClusterConfig, n config.Node) error {
 		return errors.Wrap(err, "save node")
 	}
 
-	_, err := Start(*cc, n, nil, false)
+	r, p, m, h, err := Provision(cc, &n, false)
+	if err != nil {
+		return err
+	}
+	s := Starter{
+		Runner:         r,
+		PreExists:      p,
+		MachineAPI:     m,
+		Host:           h,
+		Cfg:            cc,
+		Node:           &n,
+		ExistingAddons: nil,
+	}
+
+	_, err = Start(s, false)
 	return err
 }