From daa5b3c6d480d36934d273b46f760af7d938e9bc Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Fri, 1 Apr 2022 16:24:22 +0000 Subject: [PATCH 1/2] [ws-manager] Support custom CA certs --- components/ws-manager-api/go/config/config.go | 3 + components/ws-manager/pkg/manager/create.go | 57 +++- .../ws-manager/pkg/manager/create_test.go | 2 + .../manager/testdata/cdwp_customcerts.golden | 270 ++++++++++++++++++ .../manager/testdata/cdwp_customcerts.json | 18 ++ 5 files changed, 338 insertions(+), 12 deletions(-) create mode 100644 components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden create mode 100644 components/ws-manager/pkg/manager/testdata/cdwp_customcerts.json diff --git a/components/ws-manager-api/go/config/config.go b/components/ws-manager-api/go/config/config.go index 07ed440d839d94..ce8465bfb7f710 100644 --- a/components/ws-manager-api/go/config/config.go +++ b/components/ws-manager-api/go/config/config.go @@ -75,6 +75,9 @@ type Configuration struct { InitProbe InitProbeConfiguration `json:"initProbe"` // WorkspacePodTemplate is a path to a workspace pod template YAML file WorkspacePodTemplate WorkspacePodTemplateConfiguration `json:"podTemplate,omitempty"` + // WorkspaceCACertSecret optionally names a secret which is mounted in `/etc/ssl/certs/gp-custom.crt` + // in all workspace pods. + WorkspaceCACertSecret string `json:"caCertSecret,omitempty"` // WorkspaceURLTemplate is a Go template which resolves to the external URL of the // workspace. Available fields are: // - `ID` which is the workspace ID, diff --git a/components/ws-manager/pkg/manager/create.go b/components/ws-manager/pkg/manager/create.go index b37917c643841b..2fb140cc349f38 100644 --- a/components/ws-manager/pkg/manager/create.go +++ b/components/ws-manager/pkg/manager/create.go @@ -339,6 +339,50 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext hostPathOrCreate = corev1.HostPathDirectoryOrCreate daemonVolumeName = "daemon-mount" ) + volumes := []corev1.Volume{ + workspaceVolume, + { + Name: daemonVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: filepath.Join(m.Config.WorkspaceHostPath, startContext.Request.Id+"-daemon"), + Type: &hostPathOrCreate, + }, + }, + }, + } + + // This is how we support custom CA certs in Gitpod workspaces. + // Keep workspace templates clean. + if m.Config.WorkspaceCACertSecret != "" { + const volumeName = "custom-ca-certs" + volumes = append(volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: m.Config.WorkspaceCACertSecret, + Items: []corev1.KeyToPath{ + { + Key: "ca.crt", + Path: "ca.crt", + }, + }, + }, + }, + }) + + const mountPath = "/etc/ssl/certs/gitpod-ca.crt" + workspaceContainer.VolumeMounts = append(workspaceContainer.VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + ReadOnly: true, + MountPath: mountPath, + SubPath: "ca.crt", + }) + workspaceContainer.Env = append(workspaceContainer.Env, corev1.EnvVar{ + Name: "NODE_EXTRA_CA_CERTS", + Value: mountPath, + }) + } workloadType := "regular" if startContext.Headless { @@ -389,18 +433,7 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext *workspaceContainer, }, RestartPolicy: corev1.RestartPolicyNever, - Volumes: []corev1.Volume{ - workspaceVolume, - { - Name: daemonVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: filepath.Join(m.Config.WorkspaceHostPath, startContext.Request.Id+"-daemon"), - Type: &hostPathOrCreate, - }, - }, - }, - }, + Volumes: volumes, Tolerations: []corev1.Toleration{ { Key: "node.kubernetes.io/disk-pressure", diff --git a/components/ws-manager/pkg/manager/create_test.go b/components/ws-manager/pkg/manager/create_test.go index 5c6732f42238d0..be5996fed85b7a 100644 --- a/components/ws-manager/pkg/manager/create_test.go +++ b/components/ws-manager/pkg/manager/create_test.go @@ -31,6 +31,7 @@ func TestCreateDefiniteWorkspacePod(t *testing.T) { RegularTemplate *corev1.Pod `json:"regularTemplate,omitempty"` ResourceRequests *config.ResourceConfiguration `json:"resourceRequests,omitempty"` ResourceLimits *config.ResourceConfiguration `json:"resourceLimits,omitempty"` + CACertSecret string `json:"caCertSecret,omitempty"` EnforceAffinity bool `json:"enforceAffinity,omitempty"` } @@ -46,6 +47,7 @@ func TestCreateDefiniteWorkspacePod(t *testing.T) { fixture := input.(*fixture) mgmtCfg := forTestingOnlyManagerConfig() + mgmtCfg.WorkspaceCACertSecret = fixture.CACertSecret if fixture.ResourceRequests != nil { var ( cont = mgmtCfg.Container diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden b/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden new file mode 100644 index 00000000000000..a5a91ea18aceb1 --- /dev/null +++ b/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden @@ -0,0 +1,270 @@ +{ + "reason": { + "metadata": { + "name": "ws-test", + "namespace": "default", + "creationTimestamp": null, + "labels": { + "app": "gitpod", + "component": "workspace", + "gitpod.io/networkpolicy": "default", + "gpwsman": "true", + "headless": "false", + "metaID": "foobar", + "owner": "tester", + "workspaceID": "test", + "workspaceType": "regular" + }, + "annotations": { + "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", + "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined", + "gitpod/admission": "admit_owner_only", + "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n", + "gitpod/id": "test", + "gitpod/imageSpec": "Cghzb21lLXJlZhI0ZXUuZ2NyLmlvL2dpdHBvZC1jb3JlLWRldi9idWlkL3RoZWlhLWlkZTpzb21ldmVyc2lvbg==", + "gitpod/never-ready": "true", + "gitpod/ownerToken": "%7J'[Of/8NDiWE+9F,I6^Jcj_1\u0026}-F8p", + "gitpod/servicePrefix": "foobarservice", + "gitpod/traceid": "", + "gitpod/url": "test-foobarservice-gitpod.io", + "prometheus.io/path": "/metrics", + "prometheus.io/port": "23000", + "prometheus.io/scrape": "true", + "seccomp.security.alpha.kubernetes.io/pod": "localhost/workspace-default" + }, + "finalizers": [ + "gitpod.io/finalizer" + ] + }, + "spec": { + "volumes": [ + { + "name": "vol-this-workspace", + "hostPath": { + "path": "/tmp/workspaces/test", + "type": "DirectoryOrCreate" + } + }, + { + "name": "daemon-mount", + "hostPath": { + "path": "/tmp/workspaces/test-daemon", + "type": "DirectoryOrCreate" + } + }, + { + "name": "custom-ca-certs", + "secret": { + "secretName": "some-secret", + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ] + } + } + ], + "containers": [ + { + "name": "workspace", + "image": "registry-facade:8080/remote/test", + "command": [ + "/.supervisor/workspacekit", + "ring0" + ], + "ports": [ + { + "containerPort": 23000 + } + ], + "env": [ + { + "name": "GITPOD_REPO_ROOT", + "value": "/workspace" + }, + { + "name": "GITPOD_CLI_APITOKEN", + "value": "Ab=5=rRA*9:C'T{;RRB\u003e]vK2p6`fFfrS" + }, + { + "name": "GITPOD_WORKSPACE_ID", + "value": "foobar" + }, + { + "name": "GITPOD_INSTANCE_ID", + "value": "test" + }, + { + "name": "GITPOD_THEIA_PORT", + "value": "23000" + }, + { + "name": "THEIA_WORKSPACE_ROOT", + "value": "/workspace" + }, + { + "name": "GITPOD_HOST", + "value": "gitpod.io" + }, + { + "name": "GITPOD_WORKSPACE_URL", + "value": "test-foobarservice-gitpod.io" + }, + { + "name": "THEIA_SUPERVISOR_ENDPOINT", + "value": ":22999" + }, + { + "name": "THEIA_WEBVIEW_EXTERNAL_ENDPOINT", + "value": "webview-{{hostname}}" + }, + { + "name": "THEIA_MINI_BROWSER_HOST_PATTERN", + "value": "browser-{{hostname}}" + }, + { + "name": "GITPOD_GIT_USER_NAME", + "value": "usernameGoesHere" + }, + { + "name": "GITPOD_GIT_USER_EMAIL", + "value": "some@user.com" + }, + { + "name": "GITPOD_INTERVAL", + "value": "30000" + }, + { + "name": "GITPOD_MEMORY", + "value": "999" + }, + { + "name": "NODE_EXTRA_CA_CERTS", + "value": "/etc/ssl/certs/gitpod-ca.crt" + } + ], + "resources": { + "limits": { + "cpu": "900m", + "memory": "1G" + }, + "requests": { + "cpu": "899m", + "ephemeral-storage": "5Gi", + "memory": "999M" + } + }, + "volumeMounts": [ + { + "name": "vol-this-workspace", + "mountPath": "/workspace", + "mountPropagation": "HostToContainer" + }, + { + "name": "daemon-mount", + "mountPath": "/.workspace", + "mountPropagation": "HostToContainer" + }, + { + "name": "custom-ca-certs", + "readOnly": true, + "mountPath": "/etc/ssl/certs/gitpod-ca.crt", + "subPath": "ca.crt" + } + ], + "readinessProbe": { + "httpGet": { + "path": "/_supervisor/v1/status/content/wait/true", + "port": 22999, + "scheme": "HTTP" + }, + "initialDelaySeconds": 3, + "timeoutSeconds": 1, + "periodSeconds": 1, + "successThreshold": 1, + "failureThreshold": 600 + }, + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "add": [ + "AUDIT_WRITE", + "FSETID", + "KILL", + "NET_BIND_SERVICE", + "SYS_PTRACE" + ], + "drop": [ + "SETPCAP", + "CHOWN", + "NET_RAW", + "DAC_OVERRIDE", + "FOWNER", + "SYS_CHROOT", + "SETFCAP", + "SETUID", + "SETGID" + ] + }, + "privileged": false, + "runAsUser": 33333, + "runAsGroup": 33333, + "runAsNonRoot": true, + "readOnlyRootFilesystem": false, + "allowPrivilegeEscalation": true + } + } + ], + "restartPolicy": "Never", + "serviceAccountName": "workspace", + "automountServiceAccountToken": false, + "hostname": "foobar", + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "gitpod.io/workload_workspace_regular", + "operator": "Exists" + }, + { + "key": "gitpod.io/ws-daemon_ready_ns_default", + "operator": "Exists" + }, + { + "key": "gitpod.io/registry-facade_ready_ns_default", + "operator": "Exists" + } + ] + } + ] + } + } + }, + "tolerations": [ + { + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 30 + } + ], + "enableServiceLinks": false + }, + "status": {} + } +} \ No newline at end of file diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.json b/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.json new file mode 100644 index 00000000000000..5c129f119b8047 --- /dev/null +++ b/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.json @@ -0,0 +1,18 @@ +{ + "caCertSecret": "some-secret", + "spec": { + "ideImage": { + "webRef": "eu.gcr.io/gitpod-core-dev/buid/theia-ide:someversion" + }, + "workspaceImage": "some-ref", + "initializer": { + "snapshot": { + "snapshot": "workspaces/cryptic-id-goes-herg/fd62804b-4cab-11e9-843a-4e645373048e.tar@gitpod-dev-user-christesting" + } + }, + "git": { + "username": "usernameGoesHere", + "email": "some@user.com" + } + } +} \ No newline at end of file From 162349aee8bdb54ab56e673ded5d2fdc8746ebd3 Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Mon, 4 Apr 2022 11:22:05 +0000 Subject: [PATCH 2/2] [supervisor] Mount custom CA in ring2 if it exists --- components/workspacekit/cmd/rings.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/workspacekit/cmd/rings.go b/components/workspacekit/cmd/rings.go index 8692866271cfb7..20253d57058752 100644 --- a/components/workspacekit/cmd/rings.go +++ b/components/workspacekit/cmd/rings.go @@ -582,6 +582,7 @@ var ( "/sys", "/dev", "/etc/hostname", + "/etc/ssl/certs/gitpod-ca.crt", } rejectMountPaths = map[string]struct{}{ "/etc/resolv.conf": {},