use HTTPS in gitops run S3 bucket server

When installing the dev bucket server on the cluster, it is now
configured with a self-signed certificate generated on demand. The
certificate is valid for 3 days which should be enough for a typical
`gitops run` use case.

Instead of using HTTP in the forwarding from the host machine into the
bucket server pod, the CLI now uses HTTPS so that traffic can't be
intercepted on its way between the developer's machine and the
cluster.

The certificate, after being generated, is put into a Kubernetes
Secret which is in turn mounted into the bucket server pod.

Author: makkes

Makefile

@@ -9,7 +9,7 @@ GIT_COMMIT?=$(shell which git > /dev/null && git log -n1 --pretty='%h')
 VERSION?=$(shell which git > /dev/null && git describe --always --match "v*")
 FLUX_VERSION=0.37.0
 CHART_VERSION=$(shell which yq > /dev/null && yq e '.version' charts/gitops-server/Chart.yaml)
-DEV_BUCKET_CONTAINER_IMAGE=ghcr.io/weaveworks/gitops-bucket-server@sha256:8fbb7534e772e14ea598d287a4b54a3f556416cac6621095ce45f78346fda78a
+DEV_BUCKET_CONTAINER_IMAGE?=ghcr.io/weaveworks/gitops-bucket-server@sha256:157fa617e893e3ab0239547d8f1e820664b10c849fbd652c7f8738920b842f13
 TIER=oss
 
 # Go build args
@@ -31,7 +31,7 @@ LDFLAGS?=-X github.com/weaveworks/weave-gitops/cmd/gitops/version.Branch=$(BRANC
 # Docker args
 # LDFLAGS is passed so we don't have to copy the entire .git directory into the image
 # just to get, e.g. the commit hash
-DOCKERARGS:=--build-arg FLUX_VERSION=$(FLUX_VERSION) --build-arg LDFLAGS="$(LDFLAGS)" --build-arg GIT_COMMIT=$(GIT_COMMIT)
+DOCKERARGS+=--build-arg FLUX_VERSION=$(FLUX_VERSION) --build-arg LDFLAGS="$(LDFLAGS)" --build-arg GIT_COMMIT=$(GIT_COMMIT)
 # We want to be able to reference this in builds & pushes
 DEFAULT_DOCKER_REPO=localhost:5001
 DOCKER_REGISTRY?=$(DEFAULT_DOCKER_REPO)

cmd/gitops/beta/run/cmd.go

@@ -22,8 +22,6 @@ import (
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	"github.com/fsnotify/fsnotify"
 	"github.com/manifoldco/promptui"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/spf13/cobra"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/cmderrors"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/config"
@@ -35,6 +33,7 @@ import (
 	"github.com/weaveworks/weave-gitops/pkg/run/bootstrap"
 	"github.com/weaveworks/weave-gitops/pkg/run/install"
 	"github.com/weaveworks/weave-gitops/pkg/run/watch"
+	"github.com/weaveworks/weave-gitops/pkg/s3"
 	"github.com/weaveworks/weave-gitops/pkg/validate"
 	"github.com/weaveworks/weave-gitops/pkg/version"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -615,21 +614,22 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
 
 	// ====================== Dev-bucket ======================
 	// Install dev-bucket server before everything, so that we can also forward logs to it
-	unusedPorts, err := run.GetUnusedPorts(1)
+	unusedPorts, err := run.GetUnusedPorts(2)
 	if err != nil {
 		cancel()
 		return err
 	}
 
-	devBucketPort := unusedPorts[0]
-	cancelDevBucketPortForwarding, err := watch.InstallDevBucketServer(ctx, log0, kubeClient, cfg, devBucketPort)
+	devBucketHTTPPort := unusedPorts[0]
+	devBucketHTTPSPort := unusedPorts[1]
 
+	cancelDevBucketPortForwarding, cert, err := watch.InstallDevBucketServer(ctx, log0, kubeClient, cfg, devBucketHTTPPort, devBucketHTTPSPort)
 	if err != nil {
 		cancel()
-		return err
+		return fmt.Errorf("unable to install S3 bucket server: %w", err)
 	}
 
-	log, err := logger.NewS3LogWriter(sessionName, fmt.Sprintf("localhost:%d", devBucketPort), log0)
+	log, err := logger.NewS3LogWriter(sessionName, fmt.Sprintf("localhost:%d", devBucketHTTPSPort), cert, log0)
 	if err != nil {
 		cancel()
 		return err
@@ -666,7 +666,7 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
 		Namespace:     flags.Namespace,
 		Path:          paths.TargetDir,
 		Timeout:       flags.Timeout,
-		DevBucketPort: devBucketPort,
+		DevBucketPort: devBucketHTTPPort,
 		SessionName:   sessionName,
 		Username:      username,
 	}
@@ -683,16 +683,7 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	ignorer := watch.CreateIgnorer(paths.RootDir)
-	minioClient, err := minio.New(
-		"localhost:"+strconv.Itoa(int(devBucketPort)),
-		&minio.Options{
-			Creds:        credentials.NewStaticV4("user", "doesn't matter", ""),
-			Secure:       false,
-			BucketLookup: minio.BucketLookupPath,
-		},
-	)
-
+	minioClient, err := s3.NewMinioClient("localhost:"+strconv.Itoa(int(devBucketHTTPSPort)), cert)
 	if err != nil {
 		cancel()
 		return err
@@ -705,6 +696,8 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	ignorer := watch.CreateIgnorer(paths.RootDir)
+
 	err = filepath.Walk(paths.RootDir, watch.WatchDirsForFileWalker(watcher, ignorer))
 	if err != nil {
 		cancel()
@@ -1043,12 +1036,10 @@ func runBootstrap(ctx context.Context, log logger.Logger, paths *run.Paths, mani
 
 	workloadKustomizationContentStr := string(workloadKustomizationContent)
 
-	commitFiles := []gitprovider.CommitFile{
-		gitprovider.CommitFile{
-			Path:    &workloadKustomizationPath,
-			Content: &workloadKustomizationContentStr,
-		},
-	}
+	commitFiles := []gitprovider.CommitFile{{
+		Path:    &workloadKustomizationPath,
+		Content: &workloadKustomizationContentStr,
+	}}
 
 	if len(manifests) > 0 {
 		strManifests := string(manifests)

pkg/logger/s3_log_writer.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/weaveworks/weave-gitops/pkg/s3"
 )
 
 type S3LogWriter struct {
@@ -23,16 +23,8 @@ func (l *S3LogWriter) L() logr.Logger {
 	return l.log0.L()
 }
 
-func NewS3LogWriter(id string, endpoint string, log0 Logger) (Logger, error) {
-	minioClient, err := minio.New(
-		endpoint,
-		&minio.Options{
-			Creds:        credentials.NewStaticV4("user", "doesn't matter", ""),
-			Secure:       false,
-			BucketLookup: minio.BucketLookupPath,
-		},
-	)
-
+func NewS3LogWriter(id, endpoint string, caCert []byte, log0 Logger) (Logger, error) {
+	minioClient, err := s3.NewMinioClient(endpoint, caCert)
 	if err != nil {
 		return nil, err
 	}

pkg/run/watch/install_dev_bucket_server.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/weaveworks/weave-gitops/pkg/logger"
 	"github.com/weaveworks/weave-gitops/pkg/run"
+	"github.com/weaveworks/weave-gitops/pkg/tls"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -29,11 +30,11 @@ var (
 	// The variables below are to be set by flags passed to `go build`.
 	// Examples: -X run.DevBucketContainerImage=xxxxx
 
-	DevBucketContainerImage = "ghcr.io/weaveworks/gitops-bucket-server@sha256:8fbb7534e772e14ea598d287a4b54a3f556416cac6621095ce45f78346fda78a"
+	DevBucketContainerImage = "ghcr.io/weaveworks/gitops-bucket-server:1670322194"
 )
 
 // InstallDevBucketServer installs the dev bucket server, open port forwarding, and returns a function that can be used to the port forwarding.
-func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient client.Client, config *rest.Config, devBucketPort int32) (func(), error) {
+func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient client.Client, config *rest.Config, httpPort, httpsPort int32) (func(), []byte, error) {
 	var (
 		err                error
 		devBucketAppLabels = map[string]string{
@@ -57,7 +58,7 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 	if err != nil && apierrors.IsNotFound(err) {
 		if err := kubeClient.Create(ctx, &devBucketNamespace); err != nil {
 			log.Failuref("Error creating namespace %s: %v", GitOpsRunNamespace, err.Error())
-			return nil, err
+			return nil, nil, err
 		} else {
 			log.Successf("Created namespace %s", GitOpsRunNamespace)
 		}
@@ -76,8 +77,12 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 			Type: corev1.ServiceTypeClusterIP,
 			Ports: []corev1.ServicePort{
 				{
-					Name: RunDevBucketName,
-					Port: devBucketPort,
+					Name: fmt.Sprintf("%s-http", RunDevBucketName),
+					Port: httpPort,
+				},
+				{
+					Name: fmt.Sprintf("%s-https", RunDevBucketName),
+					Port: httpsPort,
 				},
 			},
 			Selector: devBucketAppLabels,
@@ -93,14 +98,38 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 	if err != nil && apierrors.IsNotFound(err) {
 		if err := kubeClient.Create(ctx, &devBucketService); err != nil {
 			log.Failuref("Error creating service %s/%s: %v", GitOpsRunNamespace, RunDevBucketName, err.Error())
-			return nil, err
+			return nil, nil, err
 		} else {
 			log.Successf("Created service %s/%s", GitOpsRunNamespace, RunDevBucketName)
 		}
 	} else if err == nil {
 		log.Successf("Service %s/%s already existed", GitOpsRunNamespace, RunDevBucketName)
 	}
 
+	cert, err := tls.GenerateSelfSignedCertificate("localhost", fmt.Sprintf("%s.%s.svc.cluster.local", devBucketService.Name, devBucketService.Namespace))
+	if err != nil {
+		err = fmt.Errorf("failed generating self-signed certificate for dev bucket server: %w", err)
+		log.Failuref(err.Error())
+
+		return nil, nil, err
+	}
+
+	certsSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "dev-bucket-server-certs",
+			Namespace: GitOpsRunNamespace,
+			Labels:    devBucketAppLabels,
+		},
+		Data: map[string][]byte{
+			"cert.pem": cert.Cert,
+			"cert.key": cert.Key,
+		},
+	}
+	if err := kubeClient.Create(ctx, certsSecret); err != nil {
+		log.Failuref("Error creating Secret %s/%s: %v", certsSecret.Namespace, certsSecret.Name, err.Error())
+		return nil, nil, err
+	}
+
 	// create deployment
 	replicas := int32(1)
 	devBucketDeployment := appsv1.Deployment{
@@ -119,21 +148,43 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 					Labels: devBucketAppLabels,
 				},
 				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{{
+						Name: "certs",
+						VolumeSource: corev1.VolumeSource{
+							Secret: &corev1.SecretVolumeSource{
+								SecretName: "dev-bucket-server-certs",
+							},
+						},
+					}},
 					Containers: []corev1.Container{
 						{
-							Name:  RunDevBucketName,
-							Image: DevBucketContainerImage,
+							Name:            RunDevBucketName,
+							Image:           DevBucketContainerImage,
+							ImagePullPolicy: corev1.PullIfNotPresent,
 							Env: []corev1.EnvVar{
 								{Name: "MINIO_ROOT_USER", Value: "user"},
 								{Name: "MINIO_ROOT_PASSWORD", Value: "doesn't matter"},
 							},
 							Ports: []corev1.ContainerPort{
 								{
-									ContainerPort: devBucketPort,
-									HostPort:      devBucketPort,
+									ContainerPort: httpPort,
+									HostPort:      httpPort,
+								},
+								{
+									ContainerPort: httpsPort,
+									HostPort:      httpsPort,
 								},
 							},
-							Args: []string{strconv.Itoa(int(devBucketPort))},
+							Args: []string{
+								fmt.Sprintf("--http-port=%d", httpPort),
+								fmt.Sprintf("--https-port=%d", httpsPort),
+								"--cert-file=/tmp/certs/cert.pem",
+								"--key-file=/tmp/certs/cert.key",
+							},
+							VolumeMounts: []corev1.VolumeMount{{
+								Name:      "certs",
+								MountPath: "/tmp/certs",
+							}},
 						},
 					},
 					RestartPolicy: corev1.RestartPolicyAlways,
@@ -151,7 +202,7 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 	if err != nil && apierrors.IsNotFound(err) {
 		if err := kubeClient.Create(ctx, &devBucketDeployment); err != nil {
 			log.Failuref("Error creating deployment %s/%s: %v", GitOpsRunNamespace, RunDevBucketName, err.Error())
-			return nil, err
+			return nil, nil, err
 		} else {
 			log.Successf("Created deployment %s/%s", GitOpsRunNamespace, RunDevBucketName)
 		}
@@ -189,8 +240,8 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 		Name:          RunDevBucketName,
 		Namespace:     GitOpsRunNamespace,
 		Kind:          "service",
-		HostPort:      strconv.Itoa(int(devBucketPort)),
-		ContainerPort: strconv.Itoa(int(devBucketPort)),
+		HostPort:      strconv.Itoa(int(httpsPort)),
+		ContainerPort: strconv.Itoa(int(httpsPort)),
 	}
 	// get pod from specMap
 	namespacedName := types.NamespacedName{Namespace: specMap.Namespace, Name: specMap.Name}
@@ -218,10 +269,10 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
 
 		log.Successf("Port forwarding for %s is ready.", RunDevBucketName)
 
-		return cancelPortFwd, nil
+		return cancelPortFwd, cert.Cert, nil
 	}
 
-	return nil, fmt.Errorf("pod not found")
+	return nil, nil, fmt.Errorf("pod not found")
 }
 
 // UninstallDevBucketServer deletes the dev-bucket namespace.

pkg/s3/minio.go

@@ -0,0 +1,58 @@
+package s3
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+func NewMinioClient(endpoint string, caCert []byte) (*minio.Client, error) {
+	tr, err := NewTLSRoundTripper(caCert)
+	if err != nil {
+		return nil, fmt.Errorf("failed creating transport: %w", err)
+	}
+
+	return minio.New(
+		endpoint,
+		&minio.Options{
+			Creds:        credentials.NewStaticV4("user", "doesn't matter", ""),
+			Secure:       true,
+			BucketLookup: minio.BucketLookupPath,
+			Transport:    tr,
+		},
+	)
+}
+
+func NewTLSRoundTripper(caCert []byte) (http.RoundTripper, error) {
+	tr, err := minio.DefaultTransport(true)
+	if err != nil {
+		return nil, fmt.Errorf("failed creating default transport: %w", err)
+	}
+
+	tr.TLSClientConfig = &tls.Config{
+		// Can't use SSLv3 because of POODLE and BEAST
+		// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+		// Can't use TLSv1.1 because of RC4 cipher usage
+		MinVersion: tls.VersionTLS12,
+	}
+	rootCAs := mustGetSystemCertPool()
+
+	rootCAs.AppendCertsFromPEM(caCert)
+	tr.TLSClientConfig.RootCAs = rootCAs
+
+	return tr, nil
+}
+
+// mustGetSystemCertPool - return system CAs or empty pool in case of error (or windows)
+func mustGetSystemCertPool() *x509.CertPool {
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return x509.NewCertPool()
+	}
+
+	return pool
+}