Don't store the port number with the pods

Author: ahg-g

Diff for: pkg/ext-proc/backend/datastore.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"math/rand"
-	"strconv"
 	"sync"
 
 	"github.com/go-logr/logr"
@@ -150,14 +149,12 @@ func (ds *datastore) PodDelete(namespacedName types.NamespacedName) {
 }
 
 func (ds *datastore) PodAddIfNotExist(pod *corev1.Pod) bool {
-	// new pod, add to the store for probing
-	pool, _ := ds.PoolGet()
 	new := &PodMetrics{
 		NamespacedName: types.NamespacedName{
 			Name:      pod.Name,
 			Namespace: pod.Namespace,
 		},
-		Address: pod.Status.PodIP + ":" + strconv.Itoa(int(pool.Spec.TargetPortNumber)),
+		Address: pod.Status.PodIP,
 		Metrics: Metrics{
 			ActiveModels: make(map[string]int),
 		},

Diff for: pkg/ext-proc/handlers/request.go

@@ -82,21 +82,29 @@ func (s *Server) HandleRequestBody(
 	if err != nil {
 		return nil, fmt.Errorf("failed to find target pod: %w", err)
 	}
+
 	logger.V(logutil.DEFAULT).Info("Request handled",
 		"model", llmReq.Model, "targetModel", llmReq.ResolvedTargetModel, "endpoint", targetPod)
 
+	// Insert target endpoint to instruct Envoy to route requests to the specified target pod.
+	// Attach the port port number
+	pool, err := s.datastore.PoolGet()
+	if err != nil {
+		return nil, err
+	}
+	endpoint := targetPod.Address + ":" + strconv.Itoa(int(pool.Spec.TargetPortNumber))
+
 	reqCtx.Model = llmReq.Model
 	reqCtx.ResolvedTargetModel = llmReq.ResolvedTargetModel
 	reqCtx.RequestSize = len(v.RequestBody.Body)
 	reqCtx.TargetPod = targetPod.NamespacedName.String()
-	reqCtx.TargetPodAddress = targetPod.Address
+	reqCtx.TargetEndpoint = endpoint
 
-	// Insert target endpoint to instruct Envoy to route requests to the specified target pod.
 	headers := []*configPb.HeaderValueOption{
 		{
 			Header: &configPb.HeaderValue{
 				Key:      s.targetEndpointKey,
-				RawValue: []byte(targetPod.Address),
+				RawValue: []byte(endpoint),
 			},
 		},
 		// We need to update the content length header if the body is mutated, see Envoy doc:
@@ -135,7 +143,7 @@ func (s *Server) HandleRequestBody(
 			Fields: map[string]*structpb.Value{
 				s.targetEndpointKey: {
 					Kind: &structpb.Value_StringValue{
-						StringValue: targetPod.Address,
+						StringValue: endpoint,
 					},
 				},
 			},

Diff for: pkg/ext-proc/handlers/server.go

@@ -128,7 +128,7 @@ func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error {
 // RequestContext stores context information during the life time of an HTTP request.
 type RequestContext struct {
 	TargetPod                 string
-	TargetPodAddress          string
+	TargetEndpoint            string
 	Model                     string
 	ResolvedTargetModel       string
 	RequestReceivedTimestamp  time.Time