Skip to content

Commit ce655bc

Browse files
medyaghmedyagh
authoredJan 19, 2021
Merge pull request #10099 from afbjorklund/generic-2021
Add new driver "SSH" to bootstrap generic minkube clusters over ssh
2 parents 82c513f + 541193c commit ce655bc

File tree

22 files changed

+529
-11
lines changed

22 files changed

+529
-11
lines changed
 

Diff for: ‎cmd/minikube/cmd/delete.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ func deleteProfile(profile *config.Profile) error {
309309
return DeletionError{Err: delErr, Errtype: MissingProfile}
310310
}
311311

312-
if err == nil && driver.BareMetal(cc.Driver) {
312+
if err == nil && (driver.BareMetal(cc.Driver) || driver.IsSSH(cc.Driver)) {
313313
if err := uninstallKubernetes(api, *cc, cc.Nodes[0], viper.GetString(cmdcfg.Bootstrapper)); err != nil {
314314
deletionError, ok := err.(DeletionError)
315315
if ok {

Diff for: ‎cmd/minikube/cmd/start.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *
306306
os.Exit(0)
307307
}
308308

309-
if driver.IsVM(driverName) {
309+
if driver.IsVM(driverName) && !driver.IsSSH(driverName) {
310310
url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL))
311311
if err != nil {
312312
return node.Starter{}, errors.Wrap(err, "Failed to cache ISO")
@@ -851,7 +851,7 @@ func validateUser(drvName string) {
851851

852852
// memoryLimits returns the amount of memory allocated to the system and hypervisor, the return value is in MiB
853853
func memoryLimits(drvName string) (int, int, error) {
854-
info, cpuErr, memErr, diskErr := machine.CachedHostInfo()
854+
info, cpuErr, memErr, diskErr := machine.LocalHostInfo()
855855
if cpuErr != nil {
856856
klog.Warningf("could not get system cpu info while verifying memory limits, which might be okay: %v", cpuErr)
857857
}

Diff for: ‎cmd/minikube/cmd/start_flags.go

+16
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,12 @@ const (
110110
network = "network"
111111
startNamespace = "namespace"
112112
trace = "trace"
113+
sshIPAddress = "ssh-ip-address"
114+
sshSSHUser = "ssh-user"
115+
sshSSHKey = "ssh-key"
116+
sshSSHPort = "ssh-port"
117+
defaultSSHUser = "root"
118+
defaultSSHPort = 22
113119
)
114120

115121
var (
@@ -221,6 +227,12 @@ func initNetworkingFlags() {
221227
startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
222228
startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
223229
startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
230+
231+
// ssh
232+
startCmd.Flags().String(sshIPAddress, "", "IP address (ssh driver only)")
233+
startCmd.Flags().String(sshSSHUser, defaultSSHUser, "SSH user (ssh driver only)")
234+
startCmd.Flags().String(sshSSHKey, "", "SSH key (ssh driver only)")
235+
startCmd.Flags().Int(sshSSHPort, defaultSSHPort, "SSH port (ssh driver only)")
224236
}
225237

226238
// ClusterFlagValue returns the current cluster name based on flags
@@ -335,6 +347,10 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
335347
NatNicType: viper.GetString(natNicType),
336348
StartHostTimeout: viper.GetDuration(waitTimeout),
337349
ExposedPorts: viper.GetStringSlice(ports),
350+
SSHIPAddress: viper.GetString(sshIPAddress),
351+
SSHUser: viper.GetString(sshSSHUser),
352+
SSHKey: viper.GetString(sshSSHKey),
353+
SSHPort: viper.GetInt(sshSSHPort),
338354
KubernetesConfig: config.KubernetesConfig{
339355
KubernetesVersion: k8sVersion,
340356
ClusterName: ClusterFlagValue(),

Diff for: ‎pkg/drivers/ssh/ssh.go

+241
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package ssh
18+
19+
import (
20+
"fmt"
21+
"net"
22+
"os"
23+
"os/exec"
24+
"path"
25+
"strconv"
26+
"time"
27+
28+
"github.com/docker/machine/libmachine/drivers"
29+
"github.com/docker/machine/libmachine/engine"
30+
"github.com/docker/machine/libmachine/log"
31+
"github.com/docker/machine/libmachine/mcnutils"
32+
"github.com/docker/machine/libmachine/state"
33+
"github.com/pkg/errors"
34+
"k8s.io/klog/v2"
35+
pkgdrivers "k8s.io/minikube/pkg/drivers"
36+
"k8s.io/minikube/pkg/minikube/command"
37+
"k8s.io/minikube/pkg/minikube/cruntime"
38+
"k8s.io/minikube/pkg/minikube/sysinit"
39+
)
40+
41+
// Driver is a driver designed to run kubeadm w/o VM management.
42+
// https://minikube.sigs.k8s.io/docs/reference/drivers/ssh/
43+
type Driver struct {
44+
*drivers.BaseDriver
45+
*pkgdrivers.CommonDriver
46+
EnginePort int
47+
SSHKey string
48+
runtime cruntime.Manager
49+
exec command.Runner
50+
}
51+
52+
// Config is configuration for the SSH driver
53+
type Config struct {
54+
MachineName string
55+
StorePath string
56+
ContainerRuntime string
57+
}
58+
59+
const (
60+
defaultTimeout = 15 * time.Second
61+
)
62+
63+
// NewDriver creates and returns a new instance of the driver
64+
func NewDriver(c Config) *Driver {
65+
d := &Driver{
66+
EnginePort: engine.DefaultPort,
67+
BaseDriver: &drivers.BaseDriver{
68+
MachineName: c.MachineName,
69+
StorePath: c.StorePath,
70+
},
71+
}
72+
runner := command.NewSSHRunner(d)
73+
runtime, err := cruntime.New(cruntime.Config{Type: c.ContainerRuntime, Runner: runner})
74+
// Libraries shouldn't panic, but there is no way for drivers to return error :(
75+
if err != nil {
76+
klog.Fatalf("unable to create container runtime: %v", err)
77+
}
78+
d.runtime = runtime
79+
d.exec = runner
80+
return d
81+
}
82+
83+
// DriverName returns the name of the driver
84+
func (d *Driver) DriverName() string {
85+
return "ssh"
86+
}
87+
88+
func (d *Driver) GetSSHHostname() (string, error) {
89+
return d.GetIP()
90+
}
91+
92+
func (d *Driver) GetSSHUsername() string {
93+
return d.SSHUser
94+
}
95+
96+
func (d *Driver) GetSSHKeyPath() string {
97+
return d.SSHKeyPath
98+
}
99+
100+
func (d *Driver) PreCreateCheck() error {
101+
if d.SSHKey != "" {
102+
if _, err := os.Stat(d.SSHKey); os.IsNotExist(err) {
103+
return fmt.Errorf("SSH key does not exist: %q", d.SSHKey)
104+
}
105+
}
106+
107+
return nil
108+
}
109+
110+
func (d *Driver) Create() error {
111+
if d.SSHKey == "" {
112+
log.Info("No SSH key specified. Assuming an existing key at the default location.")
113+
} else {
114+
log.Info("Importing SSH key...")
115+
116+
d.SSHKeyPath = d.ResolveStorePath(path.Base(d.SSHKey))
117+
if err := copySSHKey(d.SSHKey, d.SSHKeyPath); err != nil {
118+
return err
119+
}
120+
121+
if err := copySSHKey(d.SSHKey+".pub", d.SSHKeyPath+".pub"); err != nil {
122+
log.Infof("Couldn't copy SSH public key : %s", err)
123+
}
124+
}
125+
126+
if d.runtime.Name() == "Docker" {
127+
if _, err := d.exec.RunCmd(exec.Command("sudo", "usermod", "-aG", "docker", d.GetSSHUsername())); err != nil {
128+
return errors.Wrap(err, "usermod")
129+
}
130+
}
131+
132+
log.Debugf("IP: %s", d.IPAddress)
133+
134+
return nil
135+
}
136+
137+
func (d *Driver) GetURL() (string, error) {
138+
if err := drivers.MustBeRunning(d); err != nil {
139+
return "", err
140+
}
141+
142+
ip, err := d.GetIP()
143+
if err != nil {
144+
return "", err
145+
}
146+
147+
return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, strconv.Itoa(d.EnginePort))), nil
148+
}
149+
150+
func (d *Driver) GetState() (state.State, error) {
151+
address := net.JoinHostPort(d.IPAddress, strconv.Itoa(d.SSHPort))
152+
153+
_, err := net.DialTimeout("tcp", address, defaultTimeout)
154+
if err != nil {
155+
return state.Stopped, nil
156+
}
157+
158+
return state.Running, nil
159+
}
160+
161+
// Start a host
162+
func (d *Driver) Start() error {
163+
return nil
164+
}
165+
166+
// Stop a host gracefully, including any containers that we are managing.
167+
func (d *Driver) Stop() error {
168+
if err := sysinit.New(d.exec).Stop("kubelet"); err != nil {
169+
klog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err)
170+
if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil {
171+
klog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err)
172+
}
173+
}
174+
containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
175+
if err != nil {
176+
return errors.Wrap(err, "containers")
177+
}
178+
if len(containers) > 0 {
179+
if err := d.runtime.StopContainers(containers); err != nil {
180+
return errors.Wrap(err, "stop containers")
181+
}
182+
}
183+
klog.Infof("ssh driver is stopped!")
184+
return nil
185+
}
186+
187+
// Restart a host
188+
func (d *Driver) Restart() error {
189+
if err := sysinit.New(d.exec).Restart("kubelet"); err != nil {
190+
return err
191+
}
192+
return nil
193+
}
194+
195+
// Kill stops a host forcefully, including any containers that we are managing.
196+
func (d *Driver) Kill() error {
197+
if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil {
198+
klog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err)
199+
}
200+
201+
// First try to gracefully stop containers
202+
containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
203+
if err != nil {
204+
return errors.Wrap(err, "containers")
205+
}
206+
if len(containers) == 0 {
207+
return nil
208+
}
209+
// Try to be graceful before sending SIGKILL everywhere.
210+
if err := d.runtime.StopContainers(containers); err != nil {
211+
return errors.Wrap(err, "stop")
212+
}
213+
214+
containers, err = d.runtime.ListContainers(cruntime.ListOptions{})
215+
if err != nil {
216+
return errors.Wrap(err, "containers")
217+
}
218+
if len(containers) == 0 {
219+
return nil
220+
}
221+
if err := d.runtime.KillContainers(containers); err != nil {
222+
return errors.Wrap(err, "kill")
223+
}
224+
return nil
225+
}
226+
227+
func (d *Driver) Remove() error {
228+
return nil
229+
}
230+
231+
func copySSHKey(src, dst string) error {
232+
if err := mcnutils.CopyFile(src, dst); err != nil {
233+
return fmt.Errorf("unable to copy ssh key: %s", err)
234+
}
235+
236+
if err := os.Chmod(dst, 0600); err != nil {
237+
return fmt.Errorf("unable to set permissions on the ssh key: %s", err)
238+
}
239+
240+
return nil
241+
}

Diff for: ‎pkg/minikube/cluster/ip.go

+6
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) {
4040
return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
4141
case driver.Podman:
4242
return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
43+
case driver.SSH:
44+
ip, err := host.Driver.GetIP()
45+
if err != nil {
46+
return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
47+
}
48+
return net.ParseIP(ip), nil
4349
case driver.KVM2:
4450
return net.ParseIP("192.168.39.1"), nil
4551
case driver.HyperV:

Diff for: ‎pkg/minikube/config/types.go

+4
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ type ClusterConfig struct {
6666
HostDNSResolver bool // Only used by virtualbox
6767
HostOnlyNicType string // Only used by virtualbox
6868
NatNicType string // Only used by virtualbox
69+
SSHIPAddress string // Only used by ssh driver
70+
SSHUser string // Only used by ssh driver
71+
SSHKey string // Only used by ssh driver
72+
SSHPort int // Only used by ssh driver
6973
KubernetesConfig KubernetesConfig
7074
Nodes []Node
7175
Addons map[string]bool

Diff for: ‎pkg/minikube/driver/driver.go

+13
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ const (
3838
Mock = "mock"
3939
// None driver
4040
None = "none"
41+
// SSH driver
42+
SSH = "ssh"
4143
// KVM2 driver
4244
KVM2 = "kvm2"
4345
// VirtualBox driver
@@ -55,6 +57,8 @@ const (
5557

5658
// AliasKVM is driver name alias for kvm2
5759
AliasKVM = "kvm"
60+
// AliasSSH is driver name alias for ssh
61+
AliasSSH = "generic"
5862
)
5963

6064
var (
@@ -96,6 +100,10 @@ func MachineType(name string) string {
96100
return "container"
97101
}
98102

103+
if IsSSH(name) {
104+
return "bare metal machine"
105+
}
106+
99107
if IsVM(name) {
100108
return "VM"
101109
}
@@ -143,6 +151,11 @@ func BareMetal(name string) bool {
143151
return name == None || name == Mock
144152
}
145153

154+
// IsSSH checks if the driver is ssh
155+
func IsSSH(name string) bool {
156+
return name == SSH
157+
}
158+
146159
// NeedsPortForward returns true if driver is unable provide direct IP connectivity
147160
func NeedsPortForward(name string) bool {
148161
if !IsKIC(name) {

Diff for: ‎pkg/minikube/driver/driver_darwin.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ var supportedDrivers = []string{
2626
HyperKit,
2727
VMware,
2828
Docker,
29-
Podman,
29+
SSH,
3030
}
3131

3232
func VBoxManagePath() string {

Diff for: ‎pkg/minikube/driver/driver_linux.go

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ var supportedDrivers = []string{
2929
None,
3030
Docker,
3131
Podman,
32+
SSH,
3233
}
3334

3435
// VBoxManagePath returns the path to the VBoxManage command

Diff for: ‎pkg/minikube/driver/driver_test.go

+1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ func TestMachineType(t *testing.T) {
6565
Docker: "container",
6666
Mock: "bare metal machine",
6767
None: "bare metal machine",
68+
SSH: "bare metal machine",
6869
KVM2: "VM",
6970
VirtualBox: "VM",
7071
HyperKit: "VM",

0 commit comments

Comments
 (0)
Please sign in to comment.