Skip to content

Commit afd9dcd

Browse files
Bobbins228Bobbins228
committed
Added mtls initContainer patch
1 parent 76f065f commit afd9dcd

File tree

3 files changed

+167
-47
lines changed

3 files changed

+167
-47
lines changed

‎raycluster_webhook/config/deployment.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ spec:
2323
runAsUser: 1234
2424
containers:
2525
- name: server
26-
image: quay.io/rh_ee_egallina/altwebhook:v83
26+
image: quay.io/mcampbel/webhook:v5
2727
imagePullPolicy: Always
2828
ports:
2929
- containerPort: 8443

‎raycluster_webhook/src/k8s_client.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"sigs.k8s.io/controller-runtime/pkg/client/config"
88
)
99

10-
func createClientSet() *kubernetes.Clientset{
10+
func createClientSet() *kubernetes.Clientset {
1111
k8sConfig = config.GetConfigOrDie()
1212
clientSet, err := kubernetes.NewForConfig(k8sConfig)
1313

‎raycluster_webhook/src/patches.go

+165-45
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,20 @@ package main
22

33
import (
44
"fmt"
5+
"strconv"
56

67
rayv1api "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
78
corev1 "k8s.io/api/core/v1"
89
)
910

11+
var (
12+
oauthExists bool = false
13+
initHeadExists bool = false
14+
workerHeadExists bool = false
15+
)
16+
1017
func createPatch(rayCluster *rayv1api.RayCluster) ([]patchOperation, error) {
11-
fmt.Printf("creating json patch for RayCluster")
18+
fmt.Println("creating json patch for RayCluster")
1219

1320
var patches []patchOperation
1421
rayclusterName := rayCluster.Name
@@ -20,64 +27,177 @@ func createPatch(rayCluster *rayv1api.RayCluster) ([]patchOperation, error) {
2027
return patches, nil
2128
}
2229
}
30+
// Check for the create-cert Init Containers
31+
for _, container := range rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers {
32+
if container.Name == "create-cert" {
33+
fmt.Println("Head Init Containers already exist, no patch needed")
34+
//return patches, nil
35+
initHeadExists = true
36+
}
37+
}
38+
// Check fot the create-cert Init Container WorkerGroupSpec
39+
for _, container := range rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers {
40+
if container.Name == "create-cert" {
41+
fmt.Println("Worker Init Containers already exist, no patch needed")
42+
//return patches, nil
43+
workerHeadExists = true
44+
}
45+
}
2346

24-
// New OAuth sidecar configuration based on provided example
25-
newOAuthSidecar := corev1.Container{
26-
Name: "oauth-proxy",
27-
Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366",
28-
Ports: []corev1.ContainerPort{
29-
{
30-
ContainerPort: 8443,
31-
Name: "oauth-proxy",
47+
if !oauthExists {
48+
// New OAuth sidecar configuration based on provided example
49+
newOAuthSidecar := corev1.Container{
50+
Name: "oauth-proxy",
51+
Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366",
52+
Ports: []corev1.ContainerPort{
53+
{
54+
ContainerPort: 8443,
55+
Name: "oauth-proxy",
56+
},
3257
},
33-
},
34-
Args: []string{
35-
"--https-address=:8443",
36-
"--provider=openshift",
37-
"--openshift-service-account=" + rayclusterName + "-oauth-proxy",
38-
"--upstream=http://localhost:8265",
39-
"--tls-cert=/etc/tls/private/tls.crt",
40-
"--tls-key=/etc/tls/private/tls.key",
41-
"--cookie-secret=$(COOKIE_SECRET)",
42-
"--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}",
43-
},
44-
Env: []corev1.EnvVar{
45-
{
46-
Name: "COOKIE_SECRET",
47-
ValueFrom: &corev1.EnvVarSource{
48-
SecretKeyRef: &corev1.SecretKeySelector{
49-
LocalObjectReference: corev1.LocalObjectReference{
50-
Name: rayclusterName + "-oauth-config",
58+
Args: []string{
59+
"--https-address=:8443",
60+
"--provider=openshift",
61+
"--openshift-service-account=" + rayclusterName + "-oauth-proxy",
62+
"--upstream=http://localhost:8265",
63+
"--tls-cert=/etc/tls/private/tls.crt",
64+
"--tls-key=/etc/tls/private/tls.key",
65+
"--cookie-secret=$(COOKIE_SECRET)",
66+
"--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}",
67+
},
68+
Env: []corev1.EnvVar{
69+
{
70+
Name: "COOKIE_SECRET",
71+
ValueFrom: &corev1.EnvVarSource{
72+
SecretKeyRef: &corev1.SecretKeySelector{
73+
LocalObjectReference: corev1.LocalObjectReference{
74+
Name: rayclusterName + "-oauth-config",
75+
},
76+
Key: "cookie_secret",
5177
},
52-
Key: "cookie_secret",
5378
},
5479
},
5580
},
56-
},
57-
VolumeMounts: []corev1.VolumeMount{
58-
{
59-
Name: "proxy-tls-secret",
60-
MountPath: "/etc/tls/private",
61-
ReadOnly: true,
81+
VolumeMounts: []corev1.VolumeMount{
82+
{
83+
Name: "proxy-tls-secret",
84+
MountPath: "/etc/tls/private",
85+
ReadOnly: true,
86+
},
6287
},
63-
},
88+
}
89+
90+
// Check if the service account is set, and if not, set it
91+
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" {
92+
patches = append(patches, patchOperation{
93+
Op: "add",
94+
Path: "/spec/headGroupSpec/template/spec/serviceAccountName",
95+
Value: rayclusterName + "-oauth-proxy",
96+
})
97+
}
98+
99+
// Patch to add new OAuth sidecar
100+
patches = append(patches, patchOperation{
101+
Op: "add",
102+
Path: "/spec/headGroupSpec/template/spec/containers/-",
103+
Value: newOAuthSidecar,
104+
})
105+
}
106+
107+
patches, err := mtlsPatch(rayCluster, patches, initHeadExists, workerHeadExists)
108+
if err != nil {
109+
fmt.Errorf(err.Error())
110+
return patches, err
64111
}
65112

66-
// Check if the service account is set, and if not, set it
67-
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" {
113+
return patches, nil
114+
}
115+
116+
func mtlsPatch(rayCluster *rayv1api.RayCluster, patches []patchOperation, initHeadExists bool, workerHeadExists bool) ([]patchOperation, error) {
117+
fmt.Printf("creating json patch for RayCluster initContainers")
118+
isLocalInteractive := annotationBoolVal(rayCluster, "sdk.codeflare.dev/local_interactive", false)
119+
initContainerHead := corev1.Container{}
120+
121+
key_volumes := []corev1.VolumeMount{
122+
{
123+
Name: "ca-vol",
124+
MountPath: "/home/ray/workspace/ca",
125+
ReadOnly: true,
126+
},
127+
{
128+
Name: "server-cert",
129+
MountPath: "/home/ray/workspace/tls",
130+
ReadOnly: false,
131+
},
132+
}
133+
svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace
134+
if !initHeadExists {
135+
fmt.Println("PATCHING HEAD")
136+
if isLocalInteractive {
137+
// NEED INGRESS DOMAIN FIGURE THAT OUT ( ._.)
138+
initContainerHead = corev1.Container{
139+
Name: "create-cert",
140+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
141+
Command: []string{
142+
"sh",
143+
"-c",
144+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf 'authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayCluster.Name + `-` + rayCluster.Namespace + `-head-svc.default.svc > ./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
145+
},
146+
VolumeMounts: key_volumes,
147+
}
148+
} else {
149+
fmt.Printf("Creating init head for " + rayCluster.Name)
150+
initContainerHead = corev1.Container{
151+
Name: "create-cert",
152+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
153+
Command: []string{
154+
"sh",
155+
"-c",
156+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
157+
},
158+
VolumeMounts: key_volumes,
159+
}
160+
}
161+
// Append HeadGroup init container
68162
patches = append(patches, patchOperation{
69163
Op: "add",
70-
Path: "/spec/headGroupSpec/template/spec/serviceAccountName",
71-
Value: rayclusterName + "-oauth-proxy",
164+
Path: "/spec/headGroupSpec/template/spec/initContainers/-",
165+
Value: initContainerHead,
72166
})
73167
}
74168

75-
// Patch to add new OAuth sidecar
76-
patches = append(patches, patchOperation{
77-
Op: "add",
78-
Path: "/spec/headGroupSpec/template/spec/containers/-",
79-
Value: newOAuthSidecar,
80-
})
169+
if !workerHeadExists {
170+
fmt.Println("PATCHING WORKER")
171+
initContainerWorker := corev1.Container{
172+
Name: "create-cert",
173+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
174+
Command: []string{
175+
"sh",
176+
"-c",
177+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
178+
},
179+
VolumeMounts: key_volumes,
180+
}
181+
// Append WorkerGroup init container
182+
patches = append(patches, patchOperation{
183+
Op: "add",
184+
Path: "/spec/workerGroupSpecs/0/template/spec/initContainers/-",
185+
Value: initContainerWorker,
186+
})
187+
}
81188

82189
return patches, nil
83190
}
191+
192+
func annotationBoolVal(cluster *rayv1api.RayCluster, annotation string, defaultValue bool) bool {
193+
val, exists := cluster.ObjectMeta.Annotations[annotation]
194+
if !exists || val == "" {
195+
return defaultValue
196+
}
197+
boolVal, err := strconv.ParseBool(val)
198+
if err != nil {
199+
fmt.Errorf("Could not convert annotation value to bool", "annotation", annotation, "value", val)
200+
return defaultValue
201+
}
202+
return boolVal
203+
}

0 commit comments

Comments
 (0)