Skip to content

Commit d9f9150

Browse files
KPostOfficeKPostOffice
committed
add hash to end of resource names to avoid name clash
Signed-off-by: Kevin <[email protected]>
1 parent f31a77d commit d9f9150

File tree

4 files changed

+106
-31
lines changed

4 files changed

+106
-31
lines changed

Diff for: ‎pkg/controllers/raycluster_controller.go

+59-8
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
126126
return ctrl.Result{}, client.IgnoreNotFound(err)
127127
}
128128

129+
if err := deleteDeprecatedObjects(ctx, r, cluster); err != nil {
130+
logger.Error(err, "Failed to delete deprecated objects")
131+
return ctrl.Result{RequeueAfter: requeueTime}, err
132+
}
133+
129134
if cluster.ObjectMeta.DeletionTimestamp.IsZero() {
130135
if !controllerutil.ContainsFinalizer(cluster, oAuthFinalizer) {
131136
logger.Info("Add a finalizer", "finalizer", oAuthFinalizer)
@@ -304,7 +309,7 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool {
304309
}
305310

306311
func crbNameFromCluster(cluster *rayv1.RayCluster) string {
307-
return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz}
312+
return RCCUniqueName(cluster.Name + "-" + cluster.Namespace + "-auth")
308313
}
309314

310315
func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration {
@@ -326,7 +331,7 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.Cluster
326331
}
327332

328333
func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string {
329-
return cluster.Name + "-oauth-proxy"
334+
return RCCUniqueName(cluster.Name + "-oauth-proxy")
330335
}
331336

332337
func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountApplyConfiguration {
@@ -363,11 +368,11 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu
363368
}
364369

365370
func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string {
366-
return cluster.Name + "-oauth"
371+
return RCCUniqueName(cluster.Name + "-oauth")
367372
}
368373

369374
func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string {
370-
return cluster.Name + "-proxy-tls-secret"
375+
return RCCUniqueName(cluster.Name + "-proxy-tls-secret")
371376
}
372377

373378
func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfiguration {
@@ -389,7 +394,7 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig
389394
}
390395

391396
func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string {
392-
return cluster.Name + "-oauth-config"
397+
return RCCUniqueName(cluster.Name + "-oauth-config")
393398
}
394399

395400
// desiredOAuthSecret defines the desired OAuth secret object
@@ -406,7 +411,7 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.
406411
}
407412

408413
func caSecretNameFromCluster(cluster *rayv1.RayCluster) string {
409-
return "ca-secret-" + cluster.Name
414+
return RCCUniqueName(cluster.Name + "-ca-secret")
410415
}
411416

412417
func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.SecretApplyConfiguration {
@@ -463,7 +468,9 @@ func generateCACertificate() ([]byte, []byte, error) {
463468
}
464469

465470
func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration {
466-
return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace).
471+
return networkingv1ac.NetworkPolicy(
472+
RCCUniqueName(cluster.Name+"-workers"), cluster.Namespace,
473+
).
467474
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
468475
WithSpec(networkingv1ac.NetworkPolicySpec().
469476
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})).
@@ -484,7 +491,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf
484491
if ptr.Deref(cfg.MTLSEnabled, true) {
485492
allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)))
486493
}
487-
return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace).
494+
return networkingv1ac.NetworkPolicy(RCCUniqueName(cluster.Name+"-head"), cluster.Namespace).
488495
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
489496
WithSpec(networkingv1ac.NetworkPolicySpec().
490497
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})).
@@ -619,3 +626,47 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
619626

620627
return controller.Complete(r)
621628
}
629+
630+
func RCCUniqueName(s string) string {
631+
return s + "-" + seededHash(controllerName, s)
632+
}
633+
634+
func deleteDeprecatedObjects(ctx context.Context, r *RayClusterReconciler, cluster *rayv1.RayCluster) error {
635+
// Delete deprecated objects if they exist. These have all been replace by objects with names generated by
636+
// rccUniqueName. This is a temporary measure to clean up old objects that were created before the name generation
637+
// TODO: DELETE THIS FUNCTION AFTER A FEW RELEASES. Current release = v1.7.0 (remove in 1.9.0 or 1.10.0)
638+
logger := ctrl.LoggerFrom(ctx)
639+
if err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Delete(ctx, cluster.Name+"-oauth-config", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
640+
logger.Error(err, "Failed to delete oauth secret")
641+
return err
642+
}
643+
if err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Delete(ctx, "ca-secret-"+cluster.Name, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
644+
logger.Error(err, "Failed to delete ca secret")
645+
return err
646+
}
647+
if err := r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Delete(ctx, cluster.Name+"-oauth-proxy", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
648+
logger.Error(err, "Failed to delete SA")
649+
return err
650+
}
651+
if err := r.kubeClient.RbacV1().ClusterRoleBindings().Delete(ctx, cluster.Name+"-"+cluster.Namespace+"-auth", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
652+
logger.Error(err, "Failed to delete CRB")
653+
return err
654+
}
655+
if err := r.kubeClient.CoreV1().Services(cluster.Namespace).Delete(ctx, cluster.Name+"-oauth", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
656+
logger.Error(err, "Failed to delete service")
657+
return err
658+
}
659+
if err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Delete(ctx, cluster.Name+"-proxy-tls-secret", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
660+
logger.Error(err, "Failed to delete tls secret")
661+
return err
662+
}
663+
if err := r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Delete(ctx, cluster.Name+"-workers", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
664+
logger.Error(err, "Failed to delete worker nwp")
665+
return err
666+
}
667+
if err := r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Delete(ctx, cluster.Name+"-head", metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
668+
logger.Error(err, "Failed to delete head nwp")
669+
return err
670+
}
671+
return nil
672+
}

Diff for: ‎pkg/controllers/raycluster_webhook.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err
7474

7575
rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName))
7676

77-
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy"
77+
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = oauthServiceAccountNameFromCluster(rayCluster)
7878
}
7979

8080
if ptr.Deref(w.Config.MTLSEnabled, true) {
@@ -218,7 +218,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList {
218218
func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList {
219219
var allErrors field.ErrorList
220220

221-
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" {
221+
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != oauthServiceAccountNameFromCluster(rayCluster) {
222222
allErrors = append(allErrors, field.Invalid(
223223
field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"),
224224
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName,
@@ -241,7 +241,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
241241
ValueFrom: &corev1.EnvVarSource{
242242
SecretKeyRef: &corev1.SecretKeySelector{
243243
LocalObjectReference: corev1.LocalObjectReference{
244-
Name: rayCluster.Name + "-oauth-config",
244+
Name: oauthSecretNameFromCluster(rayCluster),
245245
},
246246
Key: "cookie_secret",
247247
},
@@ -251,7 +251,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
251251
Args: []string{
252252
"--https-address=:8443",
253253
"--provider=openshift",
254-
"--openshift-service-account=" + rayCluster.Name + "-oauth-proxy",
254+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(rayCluster),
255255
"--upstream=http://localhost:8265",
256256
"--tls-cert=/etc/tls/private/tls.crt",
257257
"--tls-key=/etc/tls/private/tls.key",
@@ -273,7 +273,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume {
273273
Name: oauthProxyVolumeName,
274274
VolumeSource: corev1.VolumeSource{
275275
Secret: &corev1.SecretVolumeSource{
276-
SecretName: rayCluster.Name + "-proxy-tls-secret",
276+
SecretName: oauthServiceTLSSecretName(rayCluster),
277277
},
278278
},
279279
}
@@ -329,7 +329,7 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
329329
Name: "ca-vol",
330330
VolumeSource: corev1.VolumeSource{
331331
Secret: &corev1.SecretVolumeSource{
332-
SecretName: `ca-secret-` + rayCluster.Name,
332+
SecretName: caSecretNameFromCluster(rayCluster),
333333
},
334334
},
335335
},
@@ -343,9 +343,9 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
343343
}
344344

345345
func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) corev1.Container {
346-
rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + config.IngressDomain
346+
rayClientRoute := rayClientNameFromCluster(rayCluster) + "-" + rayCluster.Namespace + "." + config.IngressDomain
347347
// Service name for basic interactive
348-
svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc"
348+
svcDomain := serviceNameFromCluster(rayCluster) + "." + rayCluster.Namespace + ".svc"
349349

350350
initContainerHead := corev1.Container{
351351
Name: "create-cert",

Diff for: ‎pkg/controllers/raycluster_webhook_test.go

+28-14
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func TestRayClusterWebhookDefault(t *testing.T) {
124124

125125
t.Run("Expected required service account name for the head group", func(t *testing.T) {
126126
test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName).
127-
To(Equal(validRayCluster.Name+"-oauth-proxy"),
127+
To(Equal(oauthServiceAccountNameFromCluster(validRayCluster)),
128128
"Expected the service account name to be set correctly")
129129
})
130130

@@ -230,7 +230,13 @@ func TestRayClusterWebhookDefault(t *testing.T) {
230230

231231
func TestValidateCreate(t *testing.T) {
232232
test := support.NewTest(t)
233-
233+
emptyRayCluster := &rayv1.RayCluster{
234+
ObjectMeta: metav1.ObjectMeta{
235+
Name: rayClusterName,
236+
Namespace: namespace,
237+
},
238+
Spec: rayv1.RayClusterSpec{},
239+
}
234240
validRayCluster := &rayv1.RayCluster{
235241
ObjectMeta: metav1.ObjectMeta{
236242
Name: rayClusterName,
@@ -253,7 +259,7 @@ func TestValidateCreate(t *testing.T) {
253259
ValueFrom: &corev1.EnvVarSource{
254260
SecretKeyRef: &corev1.SecretKeySelector{
255261
LocalObjectReference: corev1.LocalObjectReference{
256-
Name: rayClusterName + "-oauth-config",
262+
Name: oauthSecretNameFromCluster(emptyRayCluster),
257263
},
258264
Key: "cookie_secret",
259265
},
@@ -263,7 +269,7 @@ func TestValidateCreate(t *testing.T) {
263269
Args: []string{
264270
"--https-address=:8443",
265271
"--provider=openshift",
266-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
272+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
267273
"--upstream=http://localhost:8265",
268274
"--tls-cert=/etc/tls/private/tls.crt",
269275
"--tls-key=/etc/tls/private/tls.key",
@@ -284,12 +290,12 @@ func TestValidateCreate(t *testing.T) {
284290
Name: oauthProxyVolumeName,
285291
VolumeSource: corev1.VolumeSource{
286292
Secret: &corev1.SecretVolumeSource{
287-
SecretName: rayClusterName + "-proxy-tls-secret",
293+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
288294
},
289295
},
290296
},
291297
},
292-
ServiceAccountName: rayClusterName + "-oauth-proxy",
298+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
293299
},
294300
},
295301
RayStartParams: map[string]string{},
@@ -351,7 +357,15 @@ func TestValidateCreate(t *testing.T) {
351357

352358
func TestValidateUpdate(t *testing.T) {
353359
test := support.NewTest(t)
354-
360+
emptyRayCluster := &rayv1.RayCluster{
361+
ObjectMeta: metav1.ObjectMeta{
362+
Name: rayClusterName,
363+
Namespace: namespace,
364+
},
365+
Spec: rayv1.RayClusterSpec{},
366+
}
367+
rayClientRoute := rayClientNameFromCluster(emptyRayCluster) + "-" + emptyRayCluster.Namespace + "." + rcWebhook.Config.IngressDomain
368+
svcDomain := serviceNameFromCluster(emptyRayCluster) + "." + emptyRayCluster.Namespace + ".svc"
355369
validRayCluster := &rayv1.RayCluster{
356370
ObjectMeta: metav1.ObjectMeta{
357371
Name: rayClusterName,
@@ -374,7 +388,7 @@ func TestValidateUpdate(t *testing.T) {
374388
ValueFrom: &corev1.EnvVarSource{
375389
SecretKeyRef: &corev1.SecretKeySelector{
376390
LocalObjectReference: corev1.LocalObjectReference{
377-
Name: rayClusterName + "-oauth-config",
391+
Name: oauthSecretNameFromCluster(emptyRayCluster),
378392
},
379393
Key: "cookie_secret",
380394
},
@@ -396,7 +410,7 @@ func TestValidateUpdate(t *testing.T) {
396410
Args: []string{
397411
"--https-address=:8443",
398412
"--provider=openshift",
399-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
413+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
400414
"--upstream=http://localhost:8265",
401415
"--tls-cert=/etc/tls/private/tls.crt",
402416
"--tls-key=/etc/tls/private/tls.key",
@@ -419,7 +433,7 @@ func TestValidateUpdate(t *testing.T) {
419433
Command: []string{
420434
"sh",
421435
"-c",
422-
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayClusterName + `-` + namespace + `.\nDNS.6 = ` + rayClusterName + `-head-svc.` + namespace + `.svc` + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
436+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
423437
},
424438
VolumeMounts: []corev1.VolumeMount{
425439
{
@@ -440,15 +454,15 @@ func TestValidateUpdate(t *testing.T) {
440454
Name: oauthProxyVolumeName,
441455
VolumeSource: corev1.VolumeSource{
442456
Secret: &corev1.SecretVolumeSource{
443-
SecretName: rayClusterName + "-proxy-tls-secret",
457+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
444458
},
445459
},
446460
},
447461
{
448462
Name: "ca-vol",
449463
VolumeSource: corev1.VolumeSource{
450464
Secret: &corev1.SecretVolumeSource{
451-
SecretName: `ca-secret-` + rayClusterName,
465+
SecretName: caSecretNameFromCluster(emptyRayCluster),
452466
},
453467
},
454468
},
@@ -459,7 +473,7 @@ func TestValidateUpdate(t *testing.T) {
459473
},
460474
},
461475
},
462-
ServiceAccountName: rayClusterName + "-oauth-proxy",
476+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
463477
},
464478
},
465479
RayStartParams: map[string]string{},
@@ -505,7 +519,7 @@ func TestValidateUpdate(t *testing.T) {
505519
Name: "ca-vol",
506520
VolumeSource: corev1.VolumeSource{
507521
Secret: &corev1.SecretVolumeSource{
508-
SecretName: `ca-secret-` + rayClusterName,
522+
SecretName: caSecretNameFromCluster(emptyRayCluster),
509523
},
510524
},
511525
},

Diff for: ‎pkg/controllers/support.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package controllers
22

33
import (
4+
"crypto/sha256"
5+
"fmt"
46
"os"
57

68
"github.com/go-logr/logr"
@@ -31,7 +33,7 @@ func getEnv(key, fallback string) string {
3133
}
3234

3335
func serviceNameFromCluster(cluster *rayv1.RayCluster) string {
34-
return cluster.Name + "-head-svc"
36+
return RCCUniqueName(cluster.Name + "-head-svc")
3537
}
3638

3739
func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
@@ -209,3 +211,11 @@ func ownerRefForRayCluster(cluster *rayv1.RayCluster) *v1.OwnerReferenceApplyCon
209211
WithUID(cluster.UID).
210212
WithController(true)
211213
}
214+
215+
var (
216+
hashLength = 8
217+
)
218+
219+
func seededHash(seed string, s string) string {
220+
return fmt.Sprintf("%x", sha256.Sum256([]byte(seed+s)))[:hashLength]
221+
}

0 commit comments

Comments
 (0)