From 2c8b4edd11ceb75a6161a819cb9ed965bf3ce4ce Mon Sep 17 00:00:00 2001
From: Zhonghu Xu <xuzhonghu@huawei.com>
Date: Fri, 28 Feb 2025 09:55:12 +0800
Subject: [PATCH 1/2] make ModelName immutable and fix model weight

---
 api/v1alpha2/inferencemodel_types.go          |  3 ++-
 ...e.networking.x-k8s.io_inferencemodels.yaml |  5 ++++-
 pkg/epp/datastore/datastore.go                | 11 +++++++++--
 pkg/epp/datastore/datastore_test.go           | 19 +++++++++++++++++++
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/api/v1alpha2/inferencemodel_types.go b/api/v1alpha2/inferencemodel_types.go
index 9ab1fd86..a75fd699 100644
--- a/api/v1alpha2/inferencemodel_types.go
+++ b/api/v1alpha2/inferencemodel_types.go
@@ -71,6 +71,7 @@ type InferenceModelSpec struct {
 	//
 	// +kubebuilder:validation:MaxLength=256
 	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="modelName is immutable"
 	ModelName string `json:"modelName"`
 
 	// Criticality defines how important it is to serve the model compared to other models referencing the same pool.
@@ -175,7 +176,7 @@ type TargetModel struct {
 	// Conversely weights are optional, so long as ALL targetModels do not specify a weight.
 	//
 	// +optional
-	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Minimum=1
 	// +kubebuilder:validation:Maximum=1000000
 	Weight *int32 `json:"weight,omitempty"`
 }
diff --git a/config/crd/bases/inference.networking.x-k8s.io_inferencemodels.yaml b/config/crd/bases/inference.networking.x-k8s.io_inferencemodels.yaml
index 2995e863..63c7fb51 100644
--- a/config/crd/bases/inference.networking.x-k8s.io_inferencemodels.yaml
+++ b/config/crd/bases/inference.networking.x-k8s.io_inferencemodels.yaml
@@ -82,6 +82,9 @@ spec:
                   an error will be returned specifying that no valid target model is found.
                 maxLength: 256
                 type: string
+                x-kubernetes-validations:
+                - message: modelName is immutable
+                  rule: self == oldSelf
               poolRef:
                 description: PoolRef is a reference to the inference pool, the pool
                   must exist in the same namespace.
@@ -143,7 +146,7 @@ spec:
                         Conversely weights are optional, so long as ALL targetModels do not specify a weight.
                       format: int32
                       maximum: 1000000
-                      minimum: 0
+                      minimum: 1
                       type: integer
                   required:
                   - name
diff --git a/pkg/epp/datastore/datastore.go b/pkg/epp/datastore/datastore.go
index cd5d290f..a8120aa6 100644
--- a/pkg/epp/datastore/datastore.go
+++ b/pkg/epp/datastore/datastore.go
@@ -339,18 +339,25 @@ func stripLabelKeyAliasFromLabelMap(labels map[v1alpha2.LabelKey]v1alpha2.LabelV
 }
 
 func RandomWeightedDraw(logger logr.Logger, model *v1alpha2.InferenceModel, seed int64) string {
-	var weights int32
-
 	source := rand.NewSource(rand.Int63())
 	if seed > 0 {
 		source = rand.NewSource(seed)
 	}
 	r := rand.New(source)
+
+	// all the weight values are nil, then we should return random model name
+	if model.Spec.TargetModels[0].Weight == nil {
+		index := r.Int31n(int32(len(model.Spec.TargetModels)))
+		return model.Spec.TargetModels[index].Name
+	}
+
+	var weights int32
 	for _, model := range model.Spec.TargetModels {
 		weights += *model.Weight
 	}
 	logger.V(logutil.TRACE).Info("Weights for model computed", "model", model.Name, "weights", weights)
 	randomVal := r.Int31n(weights)
+	// TODO: optimize this without using loop
 	for _, model := range model.Spec.TargetModels {
 		if randomVal < *model.Weight {
 			return model.Name
diff --git a/pkg/epp/datastore/datastore_test.go b/pkg/epp/datastore/datastore_test.go
index edc96626..4e93b037 100644
--- a/pkg/epp/datastore/datastore_test.go
+++ b/pkg/epp/datastore/datastore_test.go
@@ -280,6 +280,25 @@ func TestRandomWeightedDraw(t *testing.T) {
 			},
 			want: "v1.1",
 		},
+		{
+			name: "weighted distribution with weight unset",
+			model: &v1alpha2.InferenceModel{
+				Spec: v1alpha2.InferenceModelSpec{
+					TargetModels: []v1alpha2.TargetModel{
+						{
+							Name: "canary",
+						},
+						{
+							Name: "v1.1",
+						},
+						{
+							Name: "v1",
+						},
+					},
+				},
+			},
+			want: "v1.1",
+		},
 	}
 	var seedVal int64 = 420
 	for _, test := range tests {

From 7ba1711738ee2034ab1f33990864301058401be7 Mon Sep 17 00:00:00 2001
From: Zhonghu Xu <xuzhonghu@huawei.com>
Date: Fri, 28 Feb 2025 10:29:40 +0800
Subject: [PATCH 2/2] Fix ut

---
 pkg/epp/datastore/datastore_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/epp/datastore/datastore_test.go b/pkg/epp/datastore/datastore_test.go
index 4e93b037..54547c55 100644
--- a/pkg/epp/datastore/datastore_test.go
+++ b/pkg/epp/datastore/datastore_test.go
@@ -297,7 +297,7 @@ func TestRandomWeightedDraw(t *testing.T) {
 					},
 				},
 			},
-			want: "v1.1",
+			want: "canary",
 		},
 	}
 	var seedVal int64 = 420
@@ -306,7 +306,7 @@ func TestRandomWeightedDraw(t *testing.T) {
 			for range 10000 {
 				model := RandomWeightedDraw(logger, test.model, seedVal)
 				if model != test.want {
-					t.Errorf("Model returned!: %v", model)
+					t.Errorf("Model returned: %v != %v", model, test.want)
 					break
 				}
 			}