Merge pull request #19383 from mfojtik/up-40-volumes

up: copy configs to remote host

Author: openshift-merge-robot

pkg/oc/bootstrap/clusteradd/cmd.go

@@ -111,7 +111,8 @@ func NewCmdAdd(name, fullName string, out, errout io.Writer) *cobra.Command {
 // Start runs the start tasks ensuring that they are executed in sequence
 func (c *ClusterAddConfig) Run() error {
 	componentsToInstall := []componentinstall.Component{}
-	installContext, err := componentinstall.NewComponentInstallContext(c.openshiftImage(), c.imageFormat(), c.BaseDir, c.ServerLogLevel)
+	installContext, err := componentinstall.NewComponentInstallContext(c.openshiftImage(), c.imageFormat(), c.BaseDir,
+		c.ServerLogLevel)
 	if err != nil {
 		return err
 	}

pkg/oc/bootstrap/clusteradd/components/registry/registry_install.go

@@ -2,11 +2,13 @@ package registry
 
 import (
 	"fmt"
+	"os"
 	"path"
 
 	"github.com/openshift/origin/pkg/oc/bootstrap/clusteradd/componentinstall"
 	"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/kubeapiserver"
 	"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper"
+	"github.com/openshift/origin/pkg/oc/bootstrap/docker/host"
 	"github.com/openshift/origin/pkg/oc/bootstrap/docker/run"
 	"github.com/openshift/origin/pkg/oc/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -48,7 +50,13 @@ func (r *RegistryComponentOptions) Install(dockerClient dockerhelper.Interface,
 		return err
 	}
 
-	masterConfigDir := path.Join(r.InstallContext.BaseDir(), kubeapiserver.KubeAPIServerDirName)
+	// If docker is on remote host, the base dir is different.
+	baseDir := r.InstallContext.BaseDir()
+	if len(os.Getenv("DOCKER_HOST")) > 0 {
+		baseDir = path.Join(host.RemoteHostOriginDir, r.InstallContext.BaseDir())
+	}
+
+	masterConfigDir := path.Join(baseDir, kubeapiserver.KubeAPIServerDirName)
 	flags := []string{
 		"adm",
 		"registry",

pkg/oc/bootstrap/clusteradd/components/router/router_install.go

@@ -11,6 +11,7 @@ import (
 	goruntime "runtime"
 
 	"github.com/golang/glog"
+	"github.com/openshift/origin/pkg/oc/bootstrap/docker/host"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -110,7 +111,21 @@ func (c *RouterComponentOptions) Install(dockerClient dockerhelper.Interface, lo
 		return errors.NewError("cannot create aggregate router cert").WithCause(err)
 	}
 
+	dockerHelper := dockerhelper.NewHelper(dockerClient)
+
+	// This snowflake makes sure that when the Docker is on remote host, we copy the generated cert into
+	// the Docker host master config dir.
+	if len(os.Getenv("DOCKER_HOST")) > 0 {
+		hostHelper := host.NewHostHelper(dockerHelper, c.InstallContext.ClientImage())
+		remoteMasterConfigDir := path.Join(host.RemoteHostOriginDir, c.InstallContext.BaseDir(), kubeapiserver.KubeAPIServerDirName)
+		if err := hostHelper.CopyToHost(masterConfigDir, remoteMasterConfigDir); err != nil {
+			return err
+		}
+		masterConfigDir = remoteMasterConfigDir
+	}
+
 	routerCertPath := masterConfigDir + "/router.pem"
+
 	flags := []string{
 		"adm", "router",
 		"--host-ports=true",

pkg/oc/bootstrap/docker/down.go

@@ -61,6 +61,7 @@ func (c *ClientStopConfig) Stop() error {
 	if err = helper.StopAndRemoveContainer(openshift.ContainerName); err != nil {
 		glog.V(2).Infof("Error stopping origin container: %v", err)
 	}
+
 	names, err := helper.ListContainerNames()
 	if err != nil {
 		return err

pkg/oc/bootstrap/docker/host/host.go

@@ -2,7 +2,10 @@ package host
 
 import (
 	"fmt"
+	"path"
 
+	"github.com/docker/docker/api/types"
+	"github.com/golang/glog"
 	"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper"
 	"github.com/openshift/origin/pkg/oc/bootstrap/docker/errors"
 	"github.com/openshift/origin/pkg/oc/bootstrap/docker/run"
@@ -17,24 +20,25 @@ nsenter --mount=/rootfs/proc/1/ns/mnt mkdir -p %[1]s
 grep -F %[1]s /rootfs/proc/1/mountinfo || nsenter --mount=/rootfs/proc/1/ns/mnt mount -o bind %[1]s %[1]s
 grep -F %[1]s /rootfs/proc/1/mountinfo | grep shared || nsenter --mount=/rootfs/proc/1/ns/mnt mount --make-shared %[1]s
 `
+
+	// RemoteHostOriginDir is a directory on the remote machine that runs Docker
+	RemoteHostOriginDir = "/var/lib/origin/cluster-up"
 )
 
 // HostHelper contains methods to help check settings on a Docker host machine
 // using a privileged container
 type HostHelper struct {
-	client     dockerhelper.Interface
-	runHelper  *run.RunHelper
-	image      string
-	volumesDir string
+	client    dockerhelper.Interface
+	runHelper *run.RunHelper
+	image     string
 }
 
 // NewHostHelper creates a new HostHelper
-func NewHostHelper(dockerHelper *dockerhelper.Helper, image, volumesDir string) *HostHelper {
+func NewHostHelper(dockerHelper *dockerhelper.Helper, image string) *HostHelper {
 	return &HostHelper{
-		runHelper:  run.NewRunHelper(dockerHelper),
-		client:     dockerHelper.Client(),
-		image:      image,
-		volumesDir: volumesDir,
+		runHelper: run.NewRunHelper(dockerHelper),
+		client:    dockerHelper.Client(),
+		image:     image,
 	}
 }
 
@@ -50,10 +54,55 @@ func (h *HostHelper) CanUseNsenterMounter() (bool, error) {
 	return err == nil && rc == 0, err
 }
 
+func (h *HostHelper) CopyToHost(src, dst string) error {
+	containerID, err := h.runner().
+		Image(h.image).
+		DiscardContainer().
+		Privileged().
+		HostPid().
+		Bind("/:/rootfs:rw").
+		Entrypoint("/bin/bash").
+		Command("-c", fmt.Sprintf("mkdir -p /rootfs/%s && sleep infinity", dst)).Start()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		h.client.ContainerStop(containerID, 1)
+		h.client.ContainerRemove(containerID, types.ContainerRemoveOptions{})
+	}()
+	err = dockerhelper.UploadFileToContainer(h.client, containerID, src, path.Join("/rootfs", dst))
+	if err != nil {
+		return err
+	}
+	glog.V(2).Infof("Succesfully copied %s to %s:%s", src, containerID, dst)
+	return nil
+}
+
+func (h *HostHelper) CopyFromHost(src, dst string) error {
+	containerID, err := h.runner().
+		Image(h.image).
+		DiscardContainer().
+		Privileged().
+		HostPid().
+		Bind("/:/rootfs:rw").
+		Entrypoint("/bin/bash").
+		Command("-c", "sleep infinity").Start()
+	if err != nil {
+		return err
+	}
+	defer h.client.ContainerStop(containerID, 1)
+	err = dockerhelper.DownloadDirFromContainer(h.client, containerID, src, path.Join("/rootfs", dst))
+	if err != nil {
+		return err
+	}
+	glog.V(2).Infof("Succesfully copied %s to %s:%s", src, containerID, dst)
+	return nil
+}
+
 // EnsureVolumeUseShareMount ensures that the host Docker VM has a shared directory that can be used
 // for OpenShift volumes. This is needed for Docker for Mac.
-func (h *HostHelper) EnsureVolumeUseShareMount() error {
-	cmd := fmt.Sprintf(ensureVolumeShareCmd, h.volumesDir)
+func (h *HostHelper) EnsureVolumeUseShareMount(volumesDir string) error {
+	cmd := fmt.Sprintf(ensureVolumeShareCmd, volumesDir)
 	_, rc, err := h.runner().
 		Image(h.image).
 		DiscardContainer().

pkg/oc/bootstrap/docker/run_self_hosted.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/openshift/origin/pkg/oc/bootstrap/docker/host"
 	kruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 
@@ -34,12 +35,6 @@ import (
 	_ "github.com/openshift/origin/pkg/api/install"
 )
 
-// This is a special case for Docker for Mac where we need to have a directory inside the VM
-// that we can re-mount with --make-shared flag. We can't use the host directories mounts as
-// they are handled by the Docker for Mac directly (via fuse.osxfs).
-// TODO: Figure out how to remove this snowflake
-const NonLinuxHostVolumeDirPrefix = "/var/lib/origin/volumes"
-
 var (
 	// staticPodLocations should only include those pods that *must* be run statically because they
 	// bring up the services required to run the workload controllers.
@@ -122,7 +117,7 @@ func (c *ClusterUpConfig) StartSelfHosted(out io.Writer) error {
 		"LOGLEVEL":                                      fmt.Sprintf("%d", c.ServerLogLevel),
 	}
 
-	clientConfigBuilder, err := kclientcmd.LoadFromFile(filepath.Join(configDirs.masterConfigDir, "admin.kubeconfig"))
+	clientConfigBuilder, err := kclientcmd.LoadFromFile(filepath.Join(c.LocalDirFor(kubeapiserver.KubeAPIServerDirName), "admin.kubeconfig"))
 	if err != nil {
 		return err
 	}
@@ -133,6 +128,7 @@ func (c *ClusterUpConfig) StartSelfHosted(out io.Writer) error {
 		return err
 	}
 
+	clientConfig.Host = c.ServerIP + ":8443"
 	// wait for the apiserver to be ready
 	glog.Info("Waiting for the kube-apiserver to be ready.")
 	if err := waitForHealthyKubeAPIServer(clientConfig); err != nil {
@@ -204,11 +200,29 @@ type configDirs struct {
 	nodeConfigDir                string
 	kubeDNSConfigDir             string
 	podManifestDir               string
+	baseDir                      string
 	err                          error
 }
 
+// LocalDirFor returns a local directory path for the given component.
+func (c *ClusterUpConfig) LocalDirFor(componentName string) string {
+	return filepath.Join(c.BaseDir, componentName)
+}
+
+// RemoteDirFor returns a directory path on remote host
+func (c *ClusterUpConfig) RemoteDirFor(componentName string) string {
+	return filepath.Join(host.RemoteHostOriginDir, c.BaseDir, componentName)
+}
+
+func (c *ClusterUpConfig) copyToRemote(source, component string) (string, error) {
+	if err := c.hostHelper.CopyToHost(source, c.RemoteDirFor(component)); err != nil {
+		return "", err
+	}
+	return c.RemoteDirFor(component), nil
+}
+
 func (c *ClusterUpConfig) BuildConfig() (*configDirs, error) {
-	configLocations := &configDirs{
+	configs := &configDirs{
 		masterConfigDir:              filepath.Join(c.BaseDir, kubeapiserver.KubeAPIServerDirName),
 		openshiftAPIServerConfigDir:  filepath.Join(c.BaseDir, kubeapiserver.OpenShiftAPIServerDirName),
 		openshiftControllerConfigDir: filepath.Join(c.BaseDir, kubeapiserver.OpenShiftControllerManagerDirName),
@@ -217,61 +231,107 @@ func (c *ClusterUpConfig) BuildConfig() (*configDirs, error) {
 		podManifestDir:               filepath.Join(c.BaseDir, kubelet.PodManifestDirName),
 	}
 
-	if _, err := os.Stat(configLocations.masterConfigDir); os.IsNotExist(err) {
+	originalMasterConfigDir := configs.masterConfigDir
+	originalNodeConfigDir := configs.nodeConfigDir
+	var err error
+
+	if _, err := os.Stat(configs.masterConfigDir); os.IsNotExist(err) {
 		_, err = c.makeMasterConfig()
 		if err != nil {
 			return nil, err
 		}
 	}
-	if _, err := os.Stat(configLocations.openshiftAPIServerConfigDir); os.IsNotExist(err) {
-		_, err = c.makeOpenShiftAPIServerConfig(configLocations.masterConfigDir)
+	if c.isRemoteDocker {
+		configs.masterConfigDir, err = c.copyToRemote(configs.masterConfigDir, kubeapiserver.KubeAPIServerDirName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if _, err := os.Stat(configs.openshiftAPIServerConfigDir); os.IsNotExist(err) {
+		_, err = c.makeOpenShiftAPIServerConfig(originalMasterConfigDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if c.isRemoteDocker {
+		configs.openshiftAPIServerConfigDir, err = c.copyToRemote(configs.openshiftAPIServerConfigDir, kubeapiserver.OpenShiftAPIServerDirName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if _, err := os.Stat(configs.openshiftControllerConfigDir); os.IsNotExist(err) {
+		_, err = c.makeOpenShiftControllerConfig(originalMasterConfigDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if c.isRemoteDocker {
+		configs.openshiftControllerConfigDir, err = c.copyToRemote(configs.openshiftControllerConfigDir, kubeapiserver.OpenShiftControllerManagerDirName)
 		if err != nil {
 			return nil, err
 		}
 	}
-	if _, err := os.Stat(configLocations.openshiftControllerConfigDir); os.IsNotExist(err) {
-		_, err = c.makeOpenShiftControllerConfig(configLocations.masterConfigDir)
+
+	if _, err := os.Stat(configs.nodeConfigDir); os.IsNotExist(err) {
+		_, err = c.makeNodeConfig(configs.masterConfigDir)
 		if err != nil {
 			return nil, err
 		}
 	}
-	if _, err := os.Stat(configLocations.nodeConfigDir); os.IsNotExist(err) {
-		_, err = c.makeNodeConfig(configLocations.masterConfigDir)
+	if c.isRemoteDocker {
+		configs.nodeConfigDir, err = c.copyToRemote(configs.nodeConfigDir, kubelet.NodeConfigDirName)
 		if err != nil {
 			return nil, err
 		}
 	}
-	if _, err := os.Stat(configLocations.kubeDNSConfigDir); os.IsNotExist(err) {
-		_, err = c.makeKubeDNSConfig(configLocations.nodeConfigDir)
+
+	if _, err := os.Stat(configs.kubeDNSConfigDir); os.IsNotExist(err) {
+		_, err = c.makeKubeDNSConfig(originalNodeConfigDir)
 		if err != nil {
 			return nil, err
 		}
+
 	}
-	if _, err := os.Stat(configLocations.podManifestDir); os.IsNotExist(err) {
-		if err := os.MkdirAll(configLocations.podManifestDir, 0755); err != nil {
+	if c.isRemoteDocker {
+		configs.kubeDNSConfigDir, err = c.copyToRemote(configs.kubeDNSConfigDir, kubelet.KubeDNSDirName)
+		if err != nil {
 			return nil, err
 		}
 	}
-	glog.V(2).Infof("configLocations = %#v", *configLocations)
 
+	if _, err := os.Stat(configs.podManifestDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(configs.podManifestDir, 0755); err != nil {
+			return nil, err
+		}
+
+	}
 	substitutions := map[string]string{
-		"/path/to/master/config-dir":              configLocations.masterConfigDir,
-		"/path/to/openshift-apiserver/config-dir": configLocations.openshiftAPIServerConfigDir,
+		"/path/to/master/config-dir":              configs.masterConfigDir,
+		"/path/to/openshift-apiserver/config-dir": configs.openshiftAPIServerConfigDir,
 		"ETCD_VOLUME":                             "emptyDir:\n",
 	}
 	if len(c.HostDataDir) > 0 {
 		substitutions["ETCD_VOLUME"] = `hostPath:
       path: ` + c.HostDataDir + "\n"
 	}
 
-	glog.V(2).Infof("Creating static pod definitions in %q", configLocations.podManifestDir)
+	glog.V(2).Infof("Creating static pod definitions in %q", configs.podManifestDir)
 	for _, staticPodLocation := range staticPodLocations {
-		if err := staticpods.UpsertStaticPod(staticPodLocation, substitutions, configLocations.podManifestDir); err != nil {
+		if err := staticpods.UpsertStaticPod(staticPodLocation, substitutions, configs.podManifestDir); err != nil {
+			return nil, err
+		}
+	}
+	if c.isRemoteDocker {
+		configs.podManifestDir, err = c.copyToRemote(configs.podManifestDir, kubelet.PodManifestDirName)
+		if err != nil {
 			return nil, err
 		}
 	}
+	glog.V(2).Infof("configLocations = %#v", *configs)
 
-	return configLocations, nil
+	return configs, nil
 }
 
 // makeMasterConfig returns the directory where a generated masterconfig lives
@@ -282,7 +342,7 @@ func (c *ClusterUpConfig) makeMasterConfig() (string, error) {
 	container.MasterImage = c.openshiftImage()
 	container.Args = []string{
 		"--write-config=/var/lib/origin/openshift.local.config",
-		"--master=127.0.0.1",
+		fmt.Sprintf("--master=%s", c.ServerIP),
 		fmt.Sprintf("--images=%s", c.imageFormat()),
 		fmt.Sprintf("--dns=0.0.0.0:%d", c.DNSPort),
 		fmt.Sprintf("--public-master=https://%s:8443", publicHost),