Skip to content

Commit 70f4778

Browse files
KPostOfficeKPostOffice
committed
add hash to end of resource names to avoid name clash
Signed-off-by: Kevin <[email protected]>
1 parent f31a77d commit 70f4778

File tree

4 files changed

+111
-25
lines changed

4 files changed

+111
-25
lines changed

Diff for: ‎pkg/controllers/raycluster_controller.go

+50-3
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"fmt"
2929
"math/big"
3030
rand2 "math/rand"
31+
"strconv"
3132
"time"
3233

3334
dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
@@ -304,7 +305,11 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool {
304305
}
305306

306307
func crbNameFromCluster(cluster *rayv1.RayCluster) string {
307-
return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz}
308+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
309+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
310+
return RCCUniqueName(cluster.Name + "-" + cluster.Namespace + "-auth")
311+
}
312+
return cluster.Name + "-" + cluster.Namespace + "-auth"
308313
}
309314

310315
func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration {
@@ -326,6 +331,10 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.Cluster
326331
}
327332

328333
func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string {
334+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
335+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
336+
return RCCUniqueName(cluster.Name + "-oauth-proxy")
337+
}
329338
return cluster.Name + "-oauth-proxy"
330339
}
331340

@@ -363,10 +372,18 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu
363372
}
364373

365374
func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string {
375+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
376+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
377+
return RCCUniqueName(cluster.Name + "-oauth")
378+
}
366379
return cluster.Name + "-oauth"
367380
}
368381

369382
func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string {
383+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
384+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
385+
return RCCUniqueName(cluster.Name + "-proxy-tls-secret")
386+
}
370387
return cluster.Name + "-proxy-tls-secret"
371388
}
372389

@@ -389,6 +406,10 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig
389406
}
390407

391408
func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string {
409+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
410+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
411+
return RCCUniqueName(cluster.Name + "-oauth-config")
412+
}
392413
return cluster.Name + "-oauth-config"
393414
}
394415

@@ -406,6 +427,10 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.
406427
}
407428

408429
func caSecretNameFromCluster(cluster *rayv1.RayCluster) string {
430+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
431+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
432+
return RCCUniqueName(cluster.Name + "-ca-secret")
433+
}
409434
return "ca-secret-" + cluster.Name
410435
}
411436

@@ -462,8 +487,18 @@ func generateCACertificate() ([]byte, []byte, error) {
462487
return privateKeyPem, certPem, nil
463488
}
464489

490+
func workerNWPNameFromCluster(cluster *rayv1.RayCluster) string {
491+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
492+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
493+
return RCCUniqueName(cluster.Name + "-workers")
494+
}
495+
return cluster.Name + "-workers"
496+
}
497+
465498
func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration {
466-
return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace).
499+
return networkingv1ac.NetworkPolicy(
500+
workerNWPNameFromCluster(cluster), cluster.Namespace,
501+
).
467502
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
468503
WithSpec(networkingv1ac.NetworkPolicySpec().
469504
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})).
@@ -477,14 +512,22 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw
477512
WithOwnerReferences(ownerRefForRayCluster(cluster))
478513
}
479514

515+
func headNWPNameFromCluster(cluster *rayv1.RayCluster) string {
516+
val, ok := cluster.GetAnnotations()[newNameAnnotation]
517+
if val, err := strconv.ParseBool(val); err == nil && ok && val {
518+
return RCCUniqueName(cluster.Name + "-head")
519+
}
520+
return cluster.Name + "-head"
521+
}
522+
480523
func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration {
481524
allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{
482525
networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)),
483526
}
484527
if ptr.Deref(cfg.MTLSEnabled, true) {
485528
allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)))
486529
}
487-
return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace).
530+
return networkingv1ac.NetworkPolicy(headNWPNameFromCluster(cluster), cluster.Namespace).
488531
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
489532
WithSpec(networkingv1ac.NetworkPolicySpec().
490533
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})).
@@ -619,3 +662,7 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
619662

620663
return controller.Complete(r)
621664
}
665+
666+
func RCCUniqueName(s string) string {
667+
return s + "-" + seededHash(controllerName, s)
668+
}

Diff for: ‎pkg/controllers/raycluster_webhook.go

+17-8
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const (
3838
oauthProxyContainerName = "oauth-proxy"
3939
oauthProxyVolumeName = "proxy-tls-secret"
4040
initContainerName = "create-cert"
41+
newNameAnnotation = "ray.openshift.ai/hashed-names"
4142
)
4243

4344
// log is for logging in this package.
@@ -68,13 +69,21 @@ var _ webhook.CustomValidator = &rayClusterWebhook{}
6869
func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error {
6970
rayCluster := obj.(*rayv1.RayCluster)
7071

72+
// add annotation to use new names
73+
annotations := rayCluster.GetAnnotations()
74+
if annotations == nil {
75+
annotations = make(map[string]string)
76+
}
77+
annotations[newNameAnnotation] = "'true'"
78+
rayCluster.Annotations = annotations
79+
println(rayCluster.GetAnnotations())
7180
if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) {
7281
rayclusterlog.V(2).Info("Adding OAuth sidecar container")
7382
rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName))
7483

7584
rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName))
7685

77-
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy"
86+
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = oauthServiceAccountNameFromCluster(rayCluster)
7887
}
7988

8089
if ptr.Deref(w.Config.MTLSEnabled, true) {
@@ -218,7 +227,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList {
218227
func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList {
219228
var allErrors field.ErrorList
220229

221-
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" {
230+
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != oauthServiceAccountNameFromCluster(rayCluster) {
222231
allErrors = append(allErrors, field.Invalid(
223232
field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"),
224233
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName,
@@ -241,7 +250,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
241250
ValueFrom: &corev1.EnvVarSource{
242251
SecretKeyRef: &corev1.SecretKeySelector{
243252
LocalObjectReference: corev1.LocalObjectReference{
244-
Name: rayCluster.Name + "-oauth-config",
253+
Name: oauthSecretNameFromCluster(rayCluster),
245254
},
246255
Key: "cookie_secret",
247256
},
@@ -251,7 +260,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
251260
Args: []string{
252261
"--https-address=:8443",
253262
"--provider=openshift",
254-
"--openshift-service-account=" + rayCluster.Name + "-oauth-proxy",
263+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(rayCluster),
255264
"--upstream=http://localhost:8265",
256265
"--tls-cert=/etc/tls/private/tls.crt",
257266
"--tls-key=/etc/tls/private/tls.key",
@@ -273,7 +282,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume {
273282
Name: oauthProxyVolumeName,
274283
VolumeSource: corev1.VolumeSource{
275284
Secret: &corev1.SecretVolumeSource{
276-
SecretName: rayCluster.Name + "-proxy-tls-secret",
285+
SecretName: oauthServiceTLSSecretName(rayCluster),
277286
},
278287
},
279288
}
@@ -329,7 +338,7 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
329338
Name: "ca-vol",
330339
VolumeSource: corev1.VolumeSource{
331340
Secret: &corev1.SecretVolumeSource{
332-
SecretName: `ca-secret-` + rayCluster.Name,
341+
SecretName: caSecretNameFromCluster(rayCluster),
333342
},
334343
},
335344
},
@@ -343,9 +352,9 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
343352
}
344353

345354
func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) corev1.Container {
346-
rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + config.IngressDomain
355+
rayClientRoute := rayClientNameFromCluster(rayCluster) + "-" + rayCluster.Namespace + "." + config.IngressDomain
347356
// Service name for basic interactive
348-
svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc"
357+
svcDomain := serviceNameFromCluster(rayCluster) + "." + rayCluster.Namespace + ".svc"
349358

350359
initContainerHead := corev1.Container{
351360
Name: "create-cert",

Diff for: ‎pkg/controllers/raycluster_webhook_test.go

+34-14
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func TestRayClusterWebhookDefault(t *testing.T) {
124124

125125
t.Run("Expected required service account name for the head group", func(t *testing.T) {
126126
test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName).
127-
To(Equal(validRayCluster.Name+"-oauth-proxy"),
127+
To(Equal(oauthServiceAccountNameFromCluster(validRayCluster)),
128128
"Expected the service account name to be set correctly")
129129
})
130130

@@ -230,7 +230,16 @@ func TestRayClusterWebhookDefault(t *testing.T) {
230230

231231
func TestValidateCreate(t *testing.T) {
232232
test := support.NewTest(t)
233-
233+
emptyRayCluster := &rayv1.RayCluster{
234+
ObjectMeta: metav1.ObjectMeta{
235+
Name: rayClusterName,
236+
Namespace: namespace,
237+
Annotations: map[string]string{
238+
newNameAnnotation: "'true'",
239+
},
240+
},
241+
Spec: rayv1.RayClusterSpec{},
242+
}
234243
validRayCluster := &rayv1.RayCluster{
235244
ObjectMeta: metav1.ObjectMeta{
236245
Name: rayClusterName,
@@ -253,7 +262,7 @@ func TestValidateCreate(t *testing.T) {
253262
ValueFrom: &corev1.EnvVarSource{
254263
SecretKeyRef: &corev1.SecretKeySelector{
255264
LocalObjectReference: corev1.LocalObjectReference{
256-
Name: rayClusterName + "-oauth-config",
265+
Name: oauthSecretNameFromCluster(emptyRayCluster),
257266
},
258267
Key: "cookie_secret",
259268
},
@@ -263,7 +272,7 @@ func TestValidateCreate(t *testing.T) {
263272
Args: []string{
264273
"--https-address=:8443",
265274
"--provider=openshift",
266-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
275+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
267276
"--upstream=http://localhost:8265",
268277
"--tls-cert=/etc/tls/private/tls.crt",
269278
"--tls-key=/etc/tls/private/tls.key",
@@ -284,12 +293,12 @@ func TestValidateCreate(t *testing.T) {
284293
Name: oauthProxyVolumeName,
285294
VolumeSource: corev1.VolumeSource{
286295
Secret: &corev1.SecretVolumeSource{
287-
SecretName: rayClusterName + "-proxy-tls-secret",
296+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
288297
},
289298
},
290299
},
291300
},
292-
ServiceAccountName: rayClusterName + "-oauth-proxy",
301+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
293302
},
294303
},
295304
RayStartParams: map[string]string{},
@@ -351,7 +360,18 @@ func TestValidateCreate(t *testing.T) {
351360

352361
func TestValidateUpdate(t *testing.T) {
353362
test := support.NewTest(t)
354-
363+
emptyRayCluster := &rayv1.RayCluster{
364+
ObjectMeta: metav1.ObjectMeta{
365+
Name: rayClusterName,
366+
Namespace: namespace,
367+
Annotations: map[string]string{
368+
newNameAnnotation: "'true'",
369+
},
370+
},
371+
Spec: rayv1.RayClusterSpec{},
372+
}
373+
rayClientRoute := rayClientNameFromCluster(emptyRayCluster) + "-" + emptyRayCluster.Namespace + "." + rcWebhook.Config.IngressDomain
374+
svcDomain := serviceNameFromCluster(emptyRayCluster) + "." + emptyRayCluster.Namespace + ".svc"
355375
validRayCluster := &rayv1.RayCluster{
356376
ObjectMeta: metav1.ObjectMeta{
357377
Name: rayClusterName,
@@ -374,7 +394,7 @@ func TestValidateUpdate(t *testing.T) {
374394
ValueFrom: &corev1.EnvVarSource{
375395
SecretKeyRef: &corev1.SecretKeySelector{
376396
LocalObjectReference: corev1.LocalObjectReference{
377-
Name: rayClusterName + "-oauth-config",
397+
Name: oauthSecretNameFromCluster(emptyRayCluster),
378398
},
379399
Key: "cookie_secret",
380400
},
@@ -396,7 +416,7 @@ func TestValidateUpdate(t *testing.T) {
396416
Args: []string{
397417
"--https-address=:8443",
398418
"--provider=openshift",
399-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
419+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
400420
"--upstream=http://localhost:8265",
401421
"--tls-cert=/etc/tls/private/tls.crt",
402422
"--tls-key=/etc/tls/private/tls.key",
@@ -419,7 +439,7 @@ func TestValidateUpdate(t *testing.T) {
419439
Command: []string{
420440
"sh",
421441
"-c",
422-
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayClusterName + `-` + namespace + `.\nDNS.6 = ` + rayClusterName + `-head-svc.` + namespace + `.svc` + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
442+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
423443
},
424444
VolumeMounts: []corev1.VolumeMount{
425445
{
@@ -440,15 +460,15 @@ func TestValidateUpdate(t *testing.T) {
440460
Name: oauthProxyVolumeName,
441461
VolumeSource: corev1.VolumeSource{
442462
Secret: &corev1.SecretVolumeSource{
443-
SecretName: rayClusterName + "-proxy-tls-secret",
463+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
444464
},
445465
},
446466
},
447467
{
448468
Name: "ca-vol",
449469
VolumeSource: corev1.VolumeSource{
450470
Secret: &corev1.SecretVolumeSource{
451-
SecretName: `ca-secret-` + rayClusterName,
471+
SecretName: caSecretNameFromCluster(emptyRayCluster),
452472
},
453473
},
454474
},
@@ -459,7 +479,7 @@ func TestValidateUpdate(t *testing.T) {
459479
},
460480
},
461481
},
462-
ServiceAccountName: rayClusterName + "-oauth-proxy",
482+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
463483
},
464484
},
465485
RayStartParams: map[string]string{},
@@ -505,7 +525,7 @@ func TestValidateUpdate(t *testing.T) {
505525
Name: "ca-vol",
506526
VolumeSource: corev1.VolumeSource{
507527
Secret: &corev1.SecretVolumeSource{
508-
SecretName: `ca-secret-` + rayClusterName,
528+
SecretName: caSecretNameFromCluster(emptyRayCluster),
509529
},
510530
},
511531
},

Diff for: ‎pkg/controllers/support.go

+10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package controllers
22

33
import (
4+
"crypto/sha256"
5+
"fmt"
46
"os"
57

68
"github.com/go-logr/logr"
@@ -209,3 +211,11 @@ func ownerRefForRayCluster(cluster *rayv1.RayCluster) *v1.OwnerReferenceApplyCon
209211
WithUID(cluster.UID).
210212
WithController(true)
211213
}
214+
215+
var (
216+
hashLength = 8
217+
)
218+
219+
func seededHash(seed string, s string) string {
220+
return fmt.Sprintf("%x", sha256.Sum256([]byte(seed+s)))[:hashLength]
221+
}

0 commit comments

Comments
 (0)