Add makefile and cloudbuild file to build and push lora-syncer

Signed-off-by: Kunjan <[email protected]>

Author: coolkp

site-src/guides/index.md

@@ -19,70 +19,6 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
    kubectl create secret generic hf-token --from-literal=token=$HF_TOKEN # Your Hugging Face Token with access to Llama2
    kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/pkg/manifests/vllm/deployment.yaml
    ```
-   **OPTIONALLY**: Enable Dynamic loading of Lora adapters.
-   
-     [Deploy sample vllm deployment with Dynamic lora adapter enabled and Lora syncer sidecar](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/tools/dynamic-lora-sidecar/deployment.yaml)
-         
-    ***Safely rollout v2 adapter***
-    
-     1. Update lora configmap
-
-        ``` yaml
-
-              apiVersion: v1
-              kind: ConfigMap
-              metadata:
-              name: dynamic-lora-config
-              data:
-              configmap.yaml: |
-                    vLLMLoRAConfig:
-                    ensureExist:   
-                       models:
-                       - id: chatbot-v1
-                          source: gs://[TEAM-A-MODELS-BUCKET]/chatbot-v1
-                       - id: chatbot-v2
-                          source: gs://[TEAM-A-MODELS-BUCKET]/chatbot-v2     
-         ```
-
-     2. Configure a canary rollout with traffic split using LLMService. In this example, 10% of traffic to the chatbot model will be sent to v2.
-
-        ``` yaml
-        model:
-           name: chatbot
-           targetModels:
-           targetModelName: chatbot-v1
-                 weight: 90
-           targetModelName: chatbot-v2
-                 weight: 10
-        ```
-            
-     3. Finish rollout by setting the traffic to the new version 100%.
-        ```yaml
-        model:
-           name: chatbot
-           targetModels:
-           targetModelName: chatbot-v2
-                 weight: 100
-        ```
-         
-     4. Remove v1 from dynamic lora configmap.
-        ```yaml
-           apiVersion: v1
-           kind: ConfigMap
-           metadata:
-           name: dynamic-lora-config
-           data:
-           configmap.yaml: |
-                 vLLMLoRAConfig:
-                 ensureExist:
-                    models:
-                    - id: chatbot-v2
-                       source: gs://[TEAM-A-MODELS-BUCKET]/chatbot-v2
-                 ensureNotExist: # Explicitly unregisters the adapter from  model servers
-                    models:
-                    - id: chatbot-v1
-                       source: gs://[TEAM-A-MODELS-BUCKET]/chatbot-v1
-        ```
 
 
 

tools/dynamic-lora-sidecar/Makefile

@@ -0,0 +1,59 @@
+IMAGE_NAME := lora-syncer
+IMAGE_REGISTRY ?= us-central1-docker.pkg.dev/k8s-staging-images/llm-instance-gateway
+IMAGE_REPO ?= $(IMAGE_REGISTRY)/$(IMAGE_NAME)
+
+GIT_TAG ?= $(shell git describe --tags --dirty --always)
+EXTRA_TAG ?= $(if $(_PULL_BASE_REF),$(_PULL_BASE_REF),main)
+IMAGE_TAG ?= $(IMAGE_REPO):$(GIT_TAG)
+EXTRA_IMAGE_TAG ?= $(IMAGE_REPO):$(EXTRA_TAG)
+
+
+PLATFORMS ?= linux/amd64
+
+
+DOCKER_BUILDX_CMD ?= docker buildx
+IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build
+IMAGE_BUILD_EXTRA_OPTS ?=
+
+# --- Targets ---
+.PHONY: image-local-build
+image-local-build:
+	BUILDER=$(shell $(DOCKER_BUILDX_CMD) create --use)
+	$(MAKE) image-build PUSH=$(PUSH)
+	$(DOCKER_BUILDX_CMD) rm $$BUILDER
+
+.PHONY: image-local-push
+image-local-push: PUSH=--push
+image-local-push: image-local-build
+
+.PHONY: image-build
+image-build:
+	$(IMAGE_BUILD_CMD) -t $(IMAGE_TAG) \
+		--platform=$(PLATFORMS) \
+		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
+		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
+		$(PUSH) \
+		$(IMAGE_BUILD_EXTRA_OPTS) ./
+
+.PHONY: image-push
+image-push: PUSH=--push
+image-push: image-build
+
+.PHONY: run
+run:
+	docker run -v $(CURDIR)/config:/config -u appuser $(IMAGE_TAG) # Use the user name
+
+.PHONY: clean
+clean:
+	docker rmi $(IMAGE_TAG) $(EXTRA_IMAGE_TAG) 2>/dev/null || true
+
+.PHONY: clean-dangling
+clean-dangling:
+	docker rmi $(docker images -f "dangling=true" -q) 2>/dev/null || true
+
+.PHONY: test
+test:
+	python -m unittest discover
+
+.PHONY: all
+all: test image-build

tools/dynamic-lora-sidecar/cloudbuild.yaml

@@ -0,0 +1,17 @@
+# See https://cloud.google.com/cloud-build/docs/build-config
+timeout: 3000s
+
+steps:
+  - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20220830-45cbff55bc
+    entrypoint: make
+    args:
+      - image-push
+    env:
+      - GIT_TAG=$_GIT_TAG
+      - EXTRA_TAG=$_PULL_BASE_REF
+      - DOCKER_BUILDX_CMD=/buildx-entrypoint
+
+substitutions:
+  _GIT_TAG: '0.0.0'       # Default value for Git tag
+  _PULL_BASE_REF: 'main'   # Default value for branch/tag
+# No options needed!