llmservice reconciler implementation

Author: kfswain

Diff for: examples/poc/README.md

@@ -0,0 +1,23 @@
+apiVersion: inference.networking.x-k8s.io/v1alpha1
+kind: LLMService
+metadata:
+  labels:
+    app.kubernetes.io/name: api
+    app.kubernetes.io/managed-by: kustomize
+  name: llmservice-sample
+spec:
+  models:
+  - name: sql-code-assist
+  - name: npc-bot
+    objective: 
+      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
+    targetModels:
+    - name: npc-bot-v1
+      weight: 50
+    - name: npc-bot-v2
+      weight: 50 	
+  poolRef:
+  - kind: LLMServerPool
+    name: test-pool
+  - name: gemini-pool
+    kind: LLMServerPool

Diff for: examples/poc/envoy-gateway-bootstrap.png

@@ -10,6 +10,7 @@ import (
 // The datastore is a local cache of relevant data for the given LLMServerPool (currently all pulled from k8s-api)
 type K8sDatastore struct {
 	LLMServerPool *v1alpha1.LLMServerPool
+	LLMServices   *sync.Map
 	Pods          *sync.Map
 }
 

Diff for: examples/poc/manifests/llmservice.yaml

@@ -0,0 +1,58 @@
+package backend
+
+import (
+	"context"
+	"strings"
+
+	"inference.networking.x-k8s.io/llm-instance-gateway/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type LLMServiceReconciler struct {
+	client.Client
+	Scheme         *runtime.Scheme
+	Record         record.EventRecorder
+	Datastore      *K8sDatastore
+	ServerPoolName string
+	Namespace      string
+}
+
+func (c *LLMServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	if req.Namespace != c.Namespace {
+		return ctrl.Result{}, nil
+	}
+	klog.V(1).Info("reconciling LLMService", req.NamespacedName)
+
+	service := &v1alpha1.LLMService{}
+	if err := c.Get(ctx, req.NamespacedName, service); err != nil {
+		klog.Error(err, "unable to get LLMServerPool")
+		return ctrl.Result{}, err
+	}
+
+	c.updateDatastore(service)
+	return ctrl.Result{}, nil
+}
+
+func (c *LLMServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.LLMService{}).
+		Complete(c)
+}
+
+func (c *LLMServiceReconciler) updateDatastore(service *v1alpha1.LLMService) {
+
+	for _, ref := range service.Spec.PoolRef {
+		if strings.Contains(strings.ToLower(ref.Kind), strings.ToLower("LLMServerPool")) && ref.Name == c.ServerPoolName {
+			c.Datastore.LLMServices.Store(service.Name, service)
+			klog.Info("Stored!!")
+			return
+		}
+	}
+	klog.Info("Deleted :(")
+	// If we get here. The service is not relevant to this pool, remove.
+	c.Datastore.LLMServices.Delete(service.Name)
+}