ext_proc.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: instance-gateway-ext-proc
  namespace: default
  labels:
    app: instance-gateway-ext-proc
spec:
  replicas: 1
  selector:
    matchLabels:
      app: instance-gateway-ext-proc
  template:
    metadata:
      labels:
        app: instance-gateway-ext-proc
    spec:
      containers:
      - name: instance-gateway-ext-proc
        image: ghcr.io/tomatillo-and-multiverse/ext-proc:demo
        args:
        #TODO: specify label selector and dynamically update pods
        - -pods
        - "vllm-78665f78c4-h4kx4,vllm-78665f78c4-hnz84"
        - -podIPs
        - "10.24.11.6:8000,10.24.5.7:8000"
        - -enable-fairness
        - "false"
        ports:
        - containerPort: 9002
      - name: curl
        image: curlimages/curl
        command: ["sleep", "3600"]
---
apiVersion: v1
kind: Service
metadata:
  name: instance-gateway-ext-proc
  namespace: default
spec:
  selector:
    app: instance-gateway-ext-proc
  ports:
    - protocol: TCP
      port: 9002
      targetPort: 9002
  type: ClusterIP
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyExtensionPolicy
metadata:
  name: ext-proc-policy
  namespace: default
spec:
  extProc:
    - backendRefs:
      - group: ""
        kind: Service
        name: instance-gateway-ext-proc
        port: 9002
      processingMode:
        request:
          body: Buffered
        response:
      messageTimeout: 5s
  targetRef:
    group: gateway.networking.k8s.io
    kind: HTTPRoute
    name: llm-route