Adding metrics handler

Author: courageJ

pkg/ext-proc/handlers/request.go

@@ -5,10 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"time"
 
 	configPb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
 	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/backend"
+	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/metrics"
 	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/scheduling"
 	klog "k8s.io/klog/v2"
 )
@@ -18,6 +20,7 @@ import (
 // Envoy sends the request body to ext proc before sending the request to the backend server.
 func (s *Server) HandleRequestBody(reqCtx *RequestContext, req *extProcPb.ProcessingRequest) (*extProcPb.ProcessingResponse, error) {
 	klog.V(3).Infof("Handling request body")
+	requestReceivedTimestamp := time.Now()
 
 	// Unmarshal request body (must be JSON).
 	v := req.Request.(*extProcPb.ProcessingRequest_RequestBody)
@@ -116,6 +119,7 @@ func (s *Server) HandleRequestBody(reqCtx *RequestContext, req *extProcPb.Proces
 			},
 		},
 	}
+	metrics.MonitorRequest(llmReq.Model, llmReq.ResolvedTargetModel, len(v.RequestBody.Body), time.Since(requestReceivedTimestamp))
 	return resp, nil
 }
 

pkg/ext-proc/main.go

@@ -19,6 +19,7 @@ import (
 	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/backend"
 	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/backend/vllm"
 	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/handlers"
+	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/metrics"
 	"inference.networking.x-k8s.io/llm-instance-gateway/pkg/ext-proc/scheduling"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -32,6 +33,8 @@ var (
 		"port",
 		9002,
 		"gRPC port")
+	metricsPort = flag.Int(
+		"metricsPort", 9090, "metrics port")
 	targetPodHeader = flag.String(
 		"targetPodHeader",
 		"target-pod",
@@ -103,6 +106,8 @@ func main() {
 		klog.Fatalf("failed to listen: %v", err)
 	}
 
+	metrics.Register()
+	go metrics.StartMetricsHandler(*metricsPort)
 	datastore := backend.NewK8sDataStore()
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{

pkg/ext-proc/metrics/metrics_handler.go

@@ -0,0 +1,29 @@
+package metrics
+
+import (
+	"net"
+	"net/http"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
+)
+
+func StartMetricsHandler(port int) {
+	klog.Info("Starting metrics HTTP handler ...")
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(
+		legacyregistry.DefaultGatherer,
+		promhttp.HandlerOpts{},
+	))
+
+	server := &http.Server{
+		Addr:    net.JoinHostPort("", strconv.Itoa(port)),
+		Handler: mux,
+	}
+	if err := server.ListenAndServe(); err != http.ErrServerClosed {
+		klog.Fatalf("failed to start metrics HTTP handler: %v", err)
+	}
+}