llmservice.yaml

apiVersion: inference.networking.x-k8s.io/v1alpha1
kind: LLMServerPool
metadata:
  labels:
  name: vllm-llama2-7b-pool
spec:
  targetPort: 8000
  modelServerSelector:
    "app": "vllm-llama2-7b-pool"
---
apiVersion: inference.networking.x-k8s.io/v1alpha1
kind: LLMService
metadata:
  labels:
    app.kubernetes.io/name: api
    app.kubernetes.io/managed-by: kustomize
  name: llmservice-sample
spec:
  models:
  - name: sql-lora
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: sql-lora
      weight: 100	
  - name: sql-lora-0
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: sql-lora-0
      weight: 100	
  - name: sql-lora-1
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: sql-lora-1
      weight: 100	
  - name: sql-lora-2
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: sql-lora-2
      weight: 100	
  - name: sql-lora-3
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: sql-lora-3
      weight: 100	
  - name: sql-lora-4
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: sql-lora-4
      weight: 100	
  - name: tweet-summary
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: tweet-summary
      weight: 100	
  - name: tweet-summary-0
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: tweet-summary-0
      weight: 100	
  - name: tweet-summary-1
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: tweet-summary-1
      weight: 100	
  - name: tweet-summary-2
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name:  tweet-summary-2
      weight: 100	
  - name:  tweet-summary-3
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: tweet-summary-3
      weight: 100	
  - name: tweet-summary-4
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: tweet-summary-4
      weight: 100	
  - name: meta-llama/Llama-2-7b-hf
    objective: 
      desiredAveragePerOutputTokenLatencyAtP95OverMultipleRequests: 50
    targetModels:
    - name: meta-llama/Llama-2-7b-hf
      weight: 100	
  poolRef:
  - kind: LLMServerPool
    name: vllm-llama2-7b-pool