patch_policy.yaml

apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyPatchPolicy
metadata:
  name: custom-response-patch-policy
  namespace: default
spec:
  targetRef:
    group: gateway.networking.k8s.io
    kind: Gateway
    name: inference-gateway
  type: JSONPatch
  jsonPatches:
    # Necessary to create a cluster of the type: ORIGINAL_DST to allow for 
    # direct pod scheduling. Which is heavily utilized in our scheduling.
    # Specifically the field `original_dst_lb_config` allows us to enable
    # `use_http_header` and `http_header_name`. 
    # Source: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
    - type: "type.googleapis.com/envoy.config.cluster.v3.Cluster"
      name: original_destination_cluster
      operation:
        op: add
        path: ""
        value:
          name: original_destination_cluster
          type: ORIGINAL_DST
          original_dst_lb_config:
            use_http_header: true
            http_header_name: "target-pod"
          connect_timeout: 6s
          lb_policy: CLUSTER_PROVIDED
          dns_lookup_family: V4_ONLY

    # The listener is required to route requests to the original destination
    # cluster we just made.
    - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
      # The listener name is of the form <GatewayNamespace>/<GatewayName>/<GatewayListenerName>
      name: default/inference-gateway/http
      operation:
        op: add
        path: "/filter_chains"
        value:
          - filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: http
                codec_type: AUTO
                route_config:
                  name: local_route
                  virtual_hosts:      
                    - name: backend
                      domains: ["*"]
                      routes:
                        - match:
                            prefix: "/"
                          route:  
                            cluster: original_destination_cluster
                            timeout: 10s
                http_filters:
                - name: envoy.filters.http.ext_proc
                  typed_config:
                    "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor
                    failure_mode_allow: false
                    grpc_service:
                      envoy_grpc:
                        # This is the cluster name as created by the EnvoyExtensionPolicy
                        # Name is of the form <CRDKind>/<GatewayNamespace>/<ExtensionPolicyName>/<IndexOfBackend>
                        cluster_name: envoyextensionpolicy/default/ext-proc-policy/0
                    processing_mode:
                      request_header_mode: "SEND"
                      response_header_mode: "SEND"
                      request_body_mode: "BUFFERED"
                      response_body_mode: "NONE"
                      request_trailer_mode: "SKIP"
                      response_trailer_mode: "SKIP"
                - name: envoy.filters.http.router
                  typed_config:
                    "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router