Skip to content

Commit a55f870

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #47 from kfswain/api-updates
Updating API and generating code
2 parents a3339ab + d6cc086 commit a55f870

9 files changed

+54
-42
lines changed

‎api/v1alpha1/llmserverpool_types.go

+10-2
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,19 @@ import (
2525
// LLMServerPoolSpec defines the desired state of LLMServerPool
2626
type LLMServerPoolSpec struct {
2727

28-
// ModelServerSelector uses label selection to watch model server pods
28+
// ModelServerSelector uses a map of label to watch model server pods
2929
// that should be included in the LLMServerPool. ModelServers should not
3030
// be with any other Service or LLMServerPool, that behavior is not supported
3131
// and will result in sub-optimal utilization.
32-
ModelServerSelector metav1.LabelSelector `json:"modelServerSelector,omitempty"`
32+
// Due to this selector being translated to a service a simple map is used instead
33+
// of: https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#LabelSelector
34+
// To avoid footshoot errors when the https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#LabelSelectorAsMap would be used.
35+
ModelServerSelector map[string]string `json:"modelServerSelector,omitempty"`
36+
37+
// TargetPort is the port number that the model servers within the pool expect
38+
// to recieve traffic from.
39+
// This maps to the TargetPort in: https://pkg.go.dev/k8s.io/api/core/v1#ServicePort
40+
TargetPort int32
3341
}
3442

3543
// LLMServerPoolStatus defines the observed state of LLMServerPool

‎api/v1alpha1/zz_generated.deepcopy.go

+7-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎client-go/applyconfiguration/api/v1alpha1/llmserverpoolspec.go

+12-10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/ext-proc/backend/datastore.go

-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
type K8sDatastore struct {
1212
LLMServerPool *v1alpha1.LLMServerPool
1313
Pods *sync.Map
14-
Port string
1514
}
1615

1716
func (ds *K8sDatastore) GetPodIPs() []string {

‎pkg/ext-proc/backend/endpointslice_reconciler.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package backend
22

33
import (
44
"context"
5+
"fmt"
56

67
discoveryv1 "k8s.io/api/discovery/v1"
78
"k8s.io/apimachinery/pkg/runtime"
@@ -49,7 +50,7 @@ func (c *EndpointSliceReconciler) updateDatastore(slice *discoveryv1.EndpointSli
4950
for _, endpoint := range slice.Endpoints {
5051
klog.V(4).Infof("Zone: %v \n endpoint: %+v \n", c.Zone, endpoint)
5152
if c.validPod(endpoint) {
52-
pod := Pod{Name: *&endpoint.TargetRef.Name, Address: endpoint.Addresses[0] + ":" + c.Datastore.Port}
53+
pod := Pod{Name: *&endpoint.TargetRef.Name, Address: endpoint.Addresses[0] + ":" + fmt.Sprint(c.Datastore.LLMServerPool.Spec.TargetPort)}
5354
podMap[pod] = true
5455
c.Datastore.Pods.Store(pod, true)
5556
}

‎pkg/ext-proc/backend/endpointslice_reconcilier_test.go

+16-6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"sync"
55
"testing"
66

7+
"inference.networking.x-k8s.io/llm-instance-gateway/api/v1alpha1"
78
v1 "k8s.io/api/core/v1"
89
discoveryv1 "k8s.io/api/discovery/v1"
910
)
@@ -25,7 +26,11 @@ func TestUpdateDatastore_EndpointSliceReconciler(t *testing.T) {
2526
name: "Add new pod",
2627
datastore: K8sDatastore{
2728
Pods: populateMap(basePod1, basePod2),
28-
Port: "8000",
29+
LLMServerPool: &v1alpha1.LLMServerPool{
30+
Spec: v1alpha1.LLMServerPoolSpec{
31+
TargetPort: int32(8000),
32+
},
33+
},
2934
},
3035
incomingSlice: &discoveryv1.EndpointSlice{
3136
Endpoints: []discoveryv1.Endpoint{
@@ -63,14 +68,17 @@ func TestUpdateDatastore_EndpointSliceReconciler(t *testing.T) {
6368
},
6469
want: K8sDatastore{
6570
Pods: populateMap(basePod1, basePod2, basePod3),
66-
Port: "8000",
6771
},
6872
},
6973
{
7074
name: "New pod, but its not ready yet. Do not add.",
7175
datastore: K8sDatastore{
7276
Pods: populateMap(basePod1, basePod2),
73-
Port: "8000",
77+
LLMServerPool: &v1alpha1.LLMServerPool{
78+
Spec: v1alpha1.LLMServerPoolSpec{
79+
TargetPort: int32(8000),
80+
},
81+
},
7482
},
7583
incomingSlice: &discoveryv1.EndpointSlice{
7684
Endpoints: []discoveryv1.Endpoint{
@@ -108,14 +116,17 @@ func TestUpdateDatastore_EndpointSliceReconciler(t *testing.T) {
108116
},
109117
want: K8sDatastore{
110118
Pods: populateMap(basePod1, basePod2),
111-
Port: "8000",
112119
},
113120
},
114121
{
115122
name: "Existing pod not ready, new pod added, and is ready",
116123
datastore: K8sDatastore{
117124
Pods: populateMap(basePod1, basePod2),
118-
Port: "8000",
125+
LLMServerPool: &v1alpha1.LLMServerPool{
126+
Spec: v1alpha1.LLMServerPoolSpec{
127+
TargetPort: int32(8000),
128+
},
129+
},
119130
},
120131
incomingSlice: &discoveryv1.EndpointSlice{
121132
Endpoints: []discoveryv1.Endpoint{
@@ -153,7 +164,6 @@ func TestUpdateDatastore_EndpointSliceReconciler(t *testing.T) {
153164
},
154165
want: K8sDatastore{
155166
Pods: populateMap(basePod3, basePod2),
156-
Port: "8000",
157167
},
158168
},
159169
}

‎pkg/ext-proc/backend/llmlserverpool_reconciler_test.go

+6-18
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ func TestUpdateDatastore_LLMServerPoolReconciler(t *testing.T) {
2020
datastore: K8sDatastore{
2121
LLMServerPool: &v1alpha1.LLMServerPool{
2222
Spec: v1alpha1.LLMServerPoolSpec{
23-
ModelServerSelector: metav1.LabelSelector{
24-
MatchLabels: map[string]string{"app": "vllm"},
25-
},
23+
ModelServerSelector: map[string]string{"app": "vllm"},
2624
},
2725
ObjectMeta: metav1.ObjectMeta{
2826
Name: "test-pool",
@@ -32,9 +30,7 @@ func TestUpdateDatastore_LLMServerPoolReconciler(t *testing.T) {
3230
},
3331
incomingServerPool: &v1alpha1.LLMServerPool{
3432
Spec: v1alpha1.LLMServerPoolSpec{
35-
ModelServerSelector: metav1.LabelSelector{
36-
MatchLabels: map[string]string{"app": "not-vllm"},
37-
},
33+
ModelServerSelector: map[string]string{"app": "not-vllm"},
3834
},
3935
ObjectMeta: metav1.ObjectMeta{
4036
Name: "test-pool",
@@ -44,9 +40,7 @@ func TestUpdateDatastore_LLMServerPoolReconciler(t *testing.T) {
4440
want: K8sDatastore{
4541
LLMServerPool: &v1alpha1.LLMServerPool{
4642
Spec: v1alpha1.LLMServerPoolSpec{
47-
ModelServerSelector: metav1.LabelSelector{
48-
MatchLabels: map[string]string{"app": "not-vllm"},
49-
},
43+
ModelServerSelector: map[string]string{"app": "not-vllm"},
5044
},
5145
ObjectMeta: metav1.ObjectMeta{
5246
Name: "test-pool",
@@ -60,9 +54,7 @@ func TestUpdateDatastore_LLMServerPoolReconciler(t *testing.T) {
6054
datastore: K8sDatastore{
6155
LLMServerPool: &v1alpha1.LLMServerPool{
6256
Spec: v1alpha1.LLMServerPoolSpec{
63-
ModelServerSelector: metav1.LabelSelector{
64-
MatchLabels: map[string]string{"app": "vllm"},
65-
},
57+
ModelServerSelector: map[string]string{"app": "vllm"},
6658
},
6759
ObjectMeta: metav1.ObjectMeta{
6860
Name: "test-pool",
@@ -72,9 +64,7 @@ func TestUpdateDatastore_LLMServerPoolReconciler(t *testing.T) {
7264
},
7365
incomingServerPool: &v1alpha1.LLMServerPool{
7466
Spec: v1alpha1.LLMServerPoolSpec{
75-
ModelServerSelector: metav1.LabelSelector{
76-
MatchLabels: map[string]string{"technically": "this-should-never-happen"},
77-
},
67+
ModelServerSelector: map[string]string{"technically": "this-should-never-happen"},
7868
},
7969
ObjectMeta: metav1.ObjectMeta{
8070
Name: "test-pool",
@@ -84,9 +74,7 @@ func TestUpdateDatastore_LLMServerPoolReconciler(t *testing.T) {
8474
want: K8sDatastore{
8575
LLMServerPool: &v1alpha1.LLMServerPool{
8676
Spec: v1alpha1.LLMServerPoolSpec{
87-
ModelServerSelector: metav1.LabelSelector{
88-
MatchLabels: map[string]string{"app": "vllm"},
89-
},
77+
ModelServerSelector: map[string]string{"app": "vllm"},
9078
},
9179
ObjectMeta: metav1.ObjectMeta{
9280
Name: "test-pool",

‎pkg/ext-proc/backend/llmserverpool_reconciler.go

-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ type LLMServerPoolReconciler struct {
2626
ServerPoolName string
2727
Namespace string
2828
Datastore *K8sDatastore
29-
Port int
3029
Zone string
3130
}
3231

‎pkg/ext-proc/main.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ var (
3636
serviceName = flag.String("serviceName", "", "Name of the service that will be used to read the endpointslices from")
3737
namespace = flag.String("namespace", "default", "The Namespace that the server pool should exist in.")
3838
zone = flag.String("zone", "", "The zone that this instance is created in. Will be passed to the corresponding endpointSlice. ")
39-
desiredPort = flag.String("desiredPort", "8000", "The port that the model server exposes")
4039
refreshPodsInterval = flag.Duration("refreshPodsInterval", 10*time.Second, "interval to refresh pods")
4140
refreshMetricsInterval = flag.Duration("refreshMetricsInterval", 50*time.Millisecond, "interval to refresh metrics")
4241
scheme = runtime.NewScheme()
@@ -71,7 +70,7 @@ func main() {
7170
klog.Fatalf("failed to listen: %v", err)
7271
}
7372

74-
datastore := &backend.K8sDatastore{LLMServerPool: &v1alpha1.LLMServerPool{}, Pods: &sync.Map{}, Port: *desiredPort}
73+
datastore := &backend.K8sDatastore{LLMServerPool: &v1alpha1.LLMServerPool{}, Pods: &sync.Map{}}
7574

7675
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
7776
Scheme: scheme,

0 commit comments

Comments
 (0)