add helm template

config/charts/inferencepool/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

config/charts/inferencepool/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v2
+name: InferencePool
+description: A Helm chart for InferencePool
+
+type: application
+
+version: 0.1.0
+
+appVersion: "0.2.0"

config/charts/inferencepool/README.md

@@ -0,0 +1,45 @@
+# InferencePool
+
+A chart to deploy an InferencePool and a corresponding EndpointPicker (epp) deployment.  
+
+
+## Install
+
+To install an InferencePool named `pool-1`  that selects from endpoints with label `app: vllm-llama2-7b` and listening on port `8000`, you can run the following command:
+
+```txt
+$ helm install pool-1 ./config/charts/inferencepool \
+  --set inferencePool.name=pool-1 \
+  --set inferencePool.selector.app=vllm-llama2-7b \
+  --set inferencePool.targetPortNumber=8000
+```
+
+where `inferencePool.targetPortNumber` is the pod that vllm backends served on and `inferencePool.selector` is the selector to match the vllm backends.
+
+## Uninstall
+
+Run the following command to uninstall the chart:
+
+```txt
+$ helm uninstall pool-1
+```
+
+## Configuration
+
+The following table list the configurable parameters of the chart.
+
+| **Parameter Name**                          | **Description**                                                                                                   |
+|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| `inferencePool.name`                        | Name for the InferencePool, and inference extension will be named as `${inferencePool.name}-epp`.                |
+| `inferencePool.targetPortNumber`            | Target port number for the vllm backends, will be used to scrape metrics by the inference extension.             |
+| `inferencePool.selector`                     | Label selector to match vllm backends managed by the inference pool.                                             |
+| `inferenceExtension.replicas`               | Number of replicas for the inference extension service. Defaults to `1`.                                           |
+| `inferenceExtension.image.name`             | Name of the container image used for the inference extension.                                                    |
+| `inferenceExtension.image.hub`              | Registry URL where the inference extension image is hosted.                                                     |
+| `inferenceExtension.image.tag`              | Image tag of the inference extension.                                                                             |
+| `inferenceExtension.image.pullPolicy`       | Image pull policy for the container. Possible values: `Always`, `IfNotPresent`, or `Never`. Defaults to `Always`. |
+| `inferenceExtension.extProcPort`            | Port where the inference extension service is served for external processing. Defaults to `9002`.                  |
+
+## Notes
+
+This chart will only deploy an InferencePool and its corresponding EndpointPicker extension. Before install the chart, please make sure that the inference extension CRDs are installed in the cluster. For more details, please refer to the [getting started guide](https://gateway-api-inference-extension.sigs.k8s.io/guides/).

config/charts/inferencepool/templates/NOTES.txt

@@ -0,0 +1 @@
+InferencePool {{ .Values.inferencePool.name }} deployed.

config/charts/inferencepool/templates/_helpers.tpl

@@ -0,0 +1,24 @@
+{{/*
+Common labels
+*/}}
+{{- define "gateway-api-inference-extension.labels" -}}
+app.kubernetes.io/name: {{ include "gateway-api-inference-extension.name" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+{{- end }}
+
+{{/*
+Inference extension name
+*/}}
+{{- define "gateway-api-inference-extension.name" -}}
+{{- $base := .Values.inferencePool.name | default "default-pool" | lower | trim | trunc 40 -}}
+{{ $base }}-epp
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "gateway-api-inference-extension.selectorLabels" -}}
+app: {{ include "gateway-api-inference-extension.name" . }}
+{{- end -}}

config/charts/inferencepool/templates/inferencepool.yaml

@@ -0,0 +1,89 @@
+apiVersion: inference.networking.x-k8s.io/v1alpha2
+kind: InferencePool
+metadata:
+  name: {{ .Values.inferencePool.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
+spec:
+  targetPortNumber: {{ .Values.inferencePool.targetPortNumber }}
+  selector:
+      {{- range $key, $value := .Values.inferencePool.selector }}
+      {{ $key }}: {{ quote $value }}
+      {{- end }}
+  extensionRef:
+    name: {{ include "gateway-api-inference-extension.name" . }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "gateway-api-inference-extension.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.inferenceExtension.replicas | default 1 }}
+  selector:
+    matchLabels:
+      {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "gateway-api-inference-extension.name" . }}
+      containers:
+      - name: epp
+        image: {{ .Values.inferenceExtension.image.hub }}/{{ .Values.inferenceExtension.image.name }}:{{ .Values.inferenceExtension.image.tag }}
+        imagePullPolicy: {{ .Values.inferenceExtension.image.pullPolicy | default "Always" }}
+        args:
+        - -poolName
+        - {{ .Values.inferencePool.name }}
+        - -poolNamespace
+        - {{ .Release.Namespace }}
+        - -v
+        - "3"
+        - -grpcPort
+        - "9002"
+        - -grpcHealthPort
+        - "9003"
+        - -metricsPort
+        - "9090"
+        ports:
+        - name: grpc
+          containerPort: 9002
+        - name: grpc-health
+          containerPort: 9003
+        - name: metrics
+          containerPort: 9090
+        livenessProbe:
+          grpc:
+            port: 9003
+            service: inference-extension
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        readinessProbe:
+          grpc:
+            port: 9003
+            service: inference-extension
+          initialDelaySeconds: 5
+          periodSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "gateway-api-inference-extension.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
+spec:
+  selector:
+    {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 4 }}
+  ports:
+    - name: grpc-ext-proc
+      protocol: TCP
+      port: {{ .Values.inferenceExtension.extProcPort | default 9002 }}
+    - name: http-metrics
+      protocol: TCP
+      port: {{ .Values.inferenceExtension.metricsPort | default 9090 }}
+  type: ClusterIP

config/charts/inferencepool/templates/rbac.yaml

@@ -0,0 +1,45 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "gateway-api-inference-extension.name" . }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
+rules:
+- apiGroups: ["inference.networking.x-k8s.io"]
+  resources: ["inferencemodels, inferencepools"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "gateway-api-inference-extension.name" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "gateway-api-inference-extension.name" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "gateway-api-inference-extension.name" . }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "gateway-api-inference-extension.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}

config/charts/inferencepool/values.yaml

@@ -0,0 +1,14 @@
+inferenceExtension:
+  replicas: 1
+  image:
+    name: epp
+    hub: us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension
+    tag: main
+    pullPolicy: Always
+  extProcPort: 9002
+
+inferencePool:
+  name: pool-1
+  targetPortNumber: 8000
+  selector:
+    app: vllm-llama2-7b