deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: llama-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: llama-server
  template:
    metadata:
      labels:
        app: llama-server
        ai.gke.io/model: LLaMA2_7B
        ai.gke.io/inference-server: vllm
        examples.ai.gke.io/source: model-garden
    spec:
      shareProcessNamespace: true
      containers:
      - name: inference-server
        image: vllm/vllm-openai:v0.6.3.post1
        resources:
          requests:
            cpu: 5
            memory: 20Gi
            ephemeral-storage: 40Gi
            nvidia.com/gpu : 1
          limits:
            cpu: 5
            memory: 20Gi
            ephemeral-storage: 40Gi
            nvidia.com/gpu : 1
        command: ["/bin/sh", "-c"]
        args:
        - vllm serve meta-llama/Llama-3.1-8B-Instruct
        - --host=0.0.0.0
        - --port=8000
        - --tensor-parallel-size=1
        - --swap-space=16
        - --gpu-memory-utilization=0.95
        - --max-model-len=2048
        - --max-num-batched-tokens=4096
        - --disable-log-stats
        - --enable-loras
        - --max-loras=5
        env:
        - name: DEPLOY_SOURCE
          value: UI_NATIVE_MODEL
        - name: MODEL_ID
          value: "Llama2-7B"
        - name: AIP_STORAGE_URI
          value: "gs://vertex-model-garden-public-us/llama2/llama2-7b-hf"
        - name: VLLM_ALLOW_RUNTIME_LORA_UPDATING
          value: "true"
        - name: HF_TOKEN
          valueFrom:
            secretKeyRef:
              name: hf-token  # The name of your Kubernetes Secret
              key: token   # The specific key within the Secret
        - name: DYNAMIC_LORA_ROLLOUT_CONFIG
          value: "/config/configmap.yaml"
        volumeMounts:
        - mountPath: /dev/shm
          name: dshm
      initContainers:
        - name: lora-adapter-syncer
          tty: true
          stdin: true 
          image: us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/lora-syncer:main
          restartPolicy: Always
          imagePullPolicy: Always
          env: 
            - name: DYNAMIC_LORA_ROLLOUT_CONFIG
              value: "/config/configmap.yaml"
          volumeMounts: # DO NOT USE subPath
          - name: config-volume
            mountPath:  /config
      volumes:
      - name: dshm
        emptyDir:
          medium: Memory
      - name: config-volume
        configMap:
          name: dynamic-lora-config

---
apiVersion: v1
kind: Service
metadata:
  name: llama-service
spec:
  selector:
    app: llama-server
  type: ClusterIP
  ports:
  - protocol: TCP
    port: 8000
    targetPort: 8000

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: dynamic-lora-config
data:
  configmap.yaml: |
      vLLMLoRAConfig:
        name: sql-loras-llama
        defaultBaseModel: meta-llama/Llama-2-7b-hf
        ensureExist:
          models:
          - id: sql-lora-v1
            source: yard1/llama-2-7b-sql-lora-test
          - id: sql-lora-v3
            source: yard1/llama-2-7b-sql-lora-test
          - id: sql-lora-v4
            source: yard1/llama-2-7b-sql-lora-test
        ensureNotExist:
          models:
          - id: sql-lora-v2
            source: yard1/llama-2-7b-sql-lora-test