feat: improved self-test capability (launch codeflare tests such that they run within the cluster)

starpit · starpit · commit 444430567281 · 2022-08-27T18:53:25.000-04:00
i.e. this provides the capability for codeflare to launch itself as a Job within a chosen cluster
diff --git a/.github/workflows/self-test.yml b/.github/workflows/self-test.yml
@@ -0,0 +1,44 @@
+# Testing of the self-test capability
+name: Self-test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  kind:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Create k8s Kind Cluster
+      uses: helm/kind-action@v1.3.0
+      with:
+        install_only: true
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+
+    - name: Setup
+      env:
+        TERM: xterm
+      run: ./tests/kind/setup-ci.sh
+
+    - name: Run Test with profile ${{ matrix.profile }}
+      env:
+        EXECUTABLE_PATH: github-actions-production
+        TERM: xterm
+        DEBUG_KUBERNETES: true
+        TEST_LOG_AGGREGATOR: true
+      run: ./deploy/self-test/run.sh
diff --git a/deploy/self-test/Dockerfile b/deploy/self-test/Dockerfile
@@ -3,22 +3,31 @@ FROM node:lts-bullseye-slim
 LABEL org.opencontainers.image.source https://github.com/project-codeflare/codeflare-cli
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV KUI_HEADLESS=true
+ENV KUI_HEADLESS_WEBPACK=true
+ENV NO_KIND=true
+ENV ROOT=/usr/local
+ENV CODEFLARE_HEADLESS_HOME=/usr/local/CodeFlare
 
-ENV GITHUB=github.com
-ENV ORG=project-codeflare
-ENV REPO=codeflare-cli
-ENV BRANCH=main
+ADD dist/headless /usr/local/CodeFlare
+ADD bin/codeflare /usr/local/bin/codeflare
+ADD node_modules/@guidebooks/store/dist/store /usr/local/CodeFlare/store
 
-RUN apt update && apt -y install git python3 python3-pip build-essential curl sudo \
+RUN apt update && apt -y install sudo curl python3 python3-pip git \
         && pip3 install --no-cache-dir  -U pip && pip3 install --no-cache-dir -U setuptools \
-        && apt -y autoclean && apt -y clean && rm -rf /var/lib/apt/lists/* \
-        && usermod -aG sudo node \
-        && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+        && apt -y clean && rm -rf /var/lib/apt/lists/* \
+        && codeflare -n util/jq \
+        && codeflare -n util/websocat \
+        && codeflare -n ml/ray/install/cli \
+        && codeflare -n openshift/oc \
+        && codeflare -n kubernetes/helm3 \
+        && codeflare -n kubernetes/kubectl
 
-USER node
+#USER codeflare
 WORKDIR /home/node
 
-ADD self-test.sh /home/node/self-test.sh
+ADD tests/kind /home/node/tests/kind
+ADD deploy/self-test/self-test.sh /home/node/self-test.sh
 
 ENTRYPOINT ["bash"]
 CMD ["./self-test.sh"]
diff --git a/deploy/self-test/run.sh b/deploy/self-test/run.sh
@@ -1,84 +1,103 @@
 #!/usr/bin/env bash
 
+#
+# This is intended only for local testing!!
+#
+# Generally speaking, self-test launches codeflare tests such that the
+# test launcher itself runs inside of a pod (a Job, actually; see
+# self-test.yaml).
+#
+# This script does so against a local kind cluster. It is helpful to
+# test the script and self-test capability, but is not intended for
+# actual self-testing in the field.
+#
+
+set -e
+set -o pipefail
+
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 
-# name of the configmap
-CM=codeflare-self-test-config
+KIND="$SCRIPTDIR"/../../tests/kind
+export SELF_TEST_IMAGE=codeflare-self-test:test
+export VARIANTS=${VARIANTS-non-gpu1}
+YAML=$(mktemp)
 
-# we will test the latest **remote** version of the current local
-# org+branch e.g. if your local branch is from org `starpit` and
-# branch `fixing-bug`, then make sure to commit and push your changes
-# to the remote, otherwise any changes you hoped to test will not be
-# picked up
-ORG=${GITHUB_REPOSITORY_OWNER-$(git config remote.$(git config branch.main.remote).url | cut -f2 -d: | cut -f1 -d/)}
-BRANCH=$(git branch --show-current)
-echo "github org=$ORG branch=$BRANCH"
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+. "${KIND}"/values.sh
 
-# if you hit ctrl+c, we will tear everything down
-trap cleanup INT
+# start kind, if needed
+function start_kind {
+    export KUBECONFIG=$("$KIND"/setup.sh)
+    echo "[Self-Test] Using KUBECONFIG=$KUBECONFIG"
+}
 
-# configure kind if needed
-function kind() {
-    if [ -n "$NEEDS_KIND" ]; then
-        export KUBECONFIG=$("$SCRIPTDIR"/../../tests/kind/setup.sh)
-    fi
+# generate self-test.yaml with variable expansion
+function yaml {
+    echo "[Self-Test] Creating self-test.yaml"
+    TEMP=$(mktemp)
+    echo 'cat <<EOF' > $TEMP
+    cat "$SCRIPTDIR"/self-test.yaml >> $TEMP
+    echo 'EOF'       >> $TEMP
+    bash $TEMP >> $YAML
+    rm $TEMP
 }
 
 # delete the job and configmap (etc.)
-function cleanup() {
+function cleanup {
+    echo "[Self-Test] Cleaning up prior deployments of self-test"
     tput setaf 5
-    kubectl delete cm $CM
-    kubectl delete -f "$SCRIPTDIR"/self-test.yaml
+    (kubectl delete -f $YAML || exit 0)
     tput sgr0
 }
 
-# create the job and configmap (etc.)
-function start() {
-    # necessary to support cloning when run as a github action; the
-    # cloning happens in self-test.sh, inside the running pod (spawned
-    # by this apply)
-    if [ -n "$GITHUB_ACTOR" ] && [ -n "$GITHUB_TOKEN" ]; then
-        GITHUB_ACTOR_PREFIX="$GITHUB_ACTOR:$GITHUB_TOKEN@"
-    fi
-
-    tput setaf 5
-    kubectl create cm $CM \
-            --from-literal=ORG=$ORG \
-            --from-literal=BRANCH=$BRANCH \
-            --from-literal=VARIANTS=$VARIANTS \
-            --from-literal=GITHUB_ACTOR_PREFIX=$GITHUB_ACTOR_PREFIX \
-            --from-literal=GITHUB_SHA=$GITHUB_SHA \
-            --from-literal=GITHUB_REF=$GITHUB_REF
+# build docker image of self-test just for this test and load it into
+# kind
+function build {
+    tput setaf 3
+    echo "[Self-Test] Building self-test image"
+    tput sgr0
+    FAST=true npm run build:docker:self-test
+    tput setaf 3
+    echo "[Self-Test] Loading image into kind image=$SELF_TEST_IMAGE cluster=$CLUSTER"
+    tput sgr0
+    kind load docker-image $SELF_TEST_IMAGE --name $CLUSTER
+}
 
-    kubectl apply -f "$SCRIPTDIR"/self-test.yaml
+function start {
+    tput setaf 3
+    echo "[Self-Test] Deploying self-test image"
     tput sgr0
+    kubectl apply -f "$YAML"
 }
 
 # wait for a pod (of the job) to be ready
 function waitForReady() {
     tput setaf 3
-    echo "Waiting for job to start"
+    echo "[Self-Test] Waiting for job to start"
     tput sgr0
-    kubectl wait pod -l job-name=codeflare-self-test --for=condition=Ready
+    kubectl wait pod -l job-name=codeflare-self-test --for=condition=Ready --timeout=240s
 }
 
 # stream out the logs of our job's pod(s)
 function logs() {
     tput setaf 3
-    echo "Streaming out job logs"
+    echo "[Self-test] Streaming out job logs"
     tput sgr0
     kubectl logs -l job-name=codeflare-self-test -f
 }
 
+# wait for the self-test to complete
 function waitForComplete() {
     tput setaf 3
-    echo "Waiting for job completion"
+    echo "[Self-test] Waiting for job completion"
     tput sgr0
-    kubectl wait pod -l job-name=codeflare-self-test --for=condition=Complete --timeout=-1s
+    kubectl wait job codeflare-self-test --for=condition=complete --timeout=-1s
 }
 
-kind
+start_kind
+yaml
 cleanup
+build
 start
 waitForReady
 logs
diff --git a/deploy/self-test/self-test.sh b/deploy/self-test/self-test.sh
@@ -5,35 +5,15 @@ set -o pipefail
 
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 
-if [ -n "$GITHUB_REF" ]; then
-    echo "Cloning org=$ORG ref=$GITHUB_REF"
-    mkdir test
-    git init
-    git remote add origin https://github.com/${ORG}/${REPO}
-    git clone -c 'remote.origin.fetch=+refs/changes/*:${GITHUB_REF%/merge}/changes/*' https://${GITHUB_ACTOR_PREFIX}${GITHUB}/${ORG}/${REPO}.git test
-    git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +${GITHUB_SHA}:${GITHUB_REF}
-else
-    echo "Cloning org=$ORG branch=$BRANCH"
-    git clone https://${GITHUB_ACTOR_PREFIX}${GITHUB}/${ORG}/${REPO}.git -b ${BRANCH} test
-fi
-cd test
-npm ci
-npm run build:headless
-
-export KUI_HEADLESS=true
-export KUI_HEADLESS_WEBPACK=true
-export CODEFLARE_HEADLESS_HOME=$PWD/dist/headless
-
-./bin/codeflare openshift/oc
-./bin/codeflare kubernetes/kubectl
+echo "Starting CodeFlare self-test"
 
 export KUBE_NS_FOR_REAL=$(kubectl get job codeflare-self-test -o custom-columns=NS:.metadata.namespace --no-headers)
 echo "Kubernetes namespace: $KUBE_NS_FOR_REAL"
 
 if [ -n "$VARIANTS" ]; then
     variants=$VARIANTS
 else
-    variants="gpu1 non-gpu"
+    variants="gpu1 non-gpu1"
 fi
 echo "Using these variants: $variants"
 
diff --git a/deploy/self-test/self-test.yaml b/deploy/self-test/self-test.yaml
@@ -9,13 +9,16 @@ metadata:
   name: codeflare-self-test-role
 rules:
 - apiGroups: [""]
-  resources: ["pods", "pods/exec", "services"]
-  verbs: ["create", "events", "get", "watch", "list"]
+  resources: ["pods", "pods/exec", "services", "events", "secrets"]
+  verbs: ["create", "delete", "get", "watch", "list"]
+- apiGroups: ["apps"]
+  resources: ["deployments", "replicasets"]
+  verbs: ["create", "delete", "get", "watch", "list"]
 - apiGroups: ["batch"]
   resources: ["jobs"]
   verbs: ["create", "get", "watch", "list"]
 - apiGroups: [""]
-  resources: ["pods/exec"]
+  resources: ["pods/exec", "pods/portforward"]
   verbs: ["create", "delete"]
 #- apiGroups: ["apps"]
 #  resources: [deployments]
@@ -39,23 +42,19 @@ kind: Job
 metadata:
   name: codeflare-self-test
 spec:
+  completions: 1
+  parallelism: 1
   template:
     spec:
       serviceAccountName: codeflare-self-test-serviceaccount
       containers:
       - name: self-test
-        image: ghcr.io/project-codeflare/codeflare-self-test:latest
-        imagePullPolicy: Always
-        resources:
-          limits:
-            cpu: 2
-            memory: 5Gi
-          requests:
-            cpu: 300m
-            memory: 5Gi
+        image: ${SELF_TEST_IMAGE-ghcr.io/project-codeflare/codeflare-self-test:${SELF_TEST_TAG-latest}}
         env:
-          - name: GUIDEBOOK_RUN_ARGS
-            value: "-V"
+          # - name: GUIDEBOOK_RUN_ARGS
+          #  value: "-V"
+          - name: VARIANTS
+            value: ${VARIANTS}
           - name: MODE
             value: development # otherwise building codeflare-cli takes a huge amount of memory
           - name: KUBE_CONTEXT_FOR_TEST
@@ -64,35 +63,5 @@ spec:
             value: default # must match with tests/kind/profiles/...
           - name: CODEFLARE_NAMESPACE_RESTRICTED # restrict use of cluster-scoped resources
             value: "true"
-          - name: GITHUB_SHA
-            valueFrom:
-              configMapKeyRef:
-                name: codeflare-self-test-config
-                key: GITHUB_SHA
-          - name: GITHUB_REF
-            valueFrom:
-              configMapKeyRef:
-                name: codeflare-self-test-config
-                key: GITHUB_REF
-          - name: GITHUB_ACTOR_PREFIX
-            valueFrom:
-              configMapKeyRef:
-                name: codeflare-self-test-config
-                key: GITHUB_ACTOR_PREFIX
-          - name: VARIANTS
-            valueFrom:
-              configMapKeyRef:
-                name: codeflare-self-test-config
-                key: VARIANTS
-          - name: ORG
-            valueFrom:
-              configMapKeyRef:
-                name: codeflare-self-test-config
-                key: ORG
-          - name: BRANCH
-            valueFrom:
-              configMapKeyRef:
-                name: codeflare-self-test-config
-                key: BRANCH
       restartPolicy: Never
   backoffLimit: 1
diff --git a/package.json b/package.json
@@ -22,8 +22,10 @@
     "prepare": "cross-env husky install",
     "build:docker:cli": "npm run build:headless-maybe && docker build -f deploy/cli/Dockerfile -t ghcr.io/project-codeflare/codeflare-cli .",
     "publish:docker:log-aggregator": "docker push ghcr.io/project-codeflare/codeflare-log-aggregator:${VERSION-latest}",
-    "build:docker:log-aggregator": "npm run build:headless-maybe && docker build -f deploy/log-aggregator/Dockerfile -t ${LOG_AGGREGATOR_IMAGE-ghcr.io/project-codeflare/codeflare-log-aggregator:${version-latest}} .",
-    "build:docker:self-test": "cd deploy/self-test && docker build -t ghcr.io/project-codeflare/codeflare-self-test .",
+    "build:docker:log-aggregator": "npm run build:headless-maybe && docker build -f deploy/log-aggregator/Dockerfile -t ${LOG_AGGREGATOR_IMAGE-ghcr.io/project-codeflare/codeflare-log-aggregator:${VERSION-latest}} .",
+    "publish:docker:self-test": "docker push ghcr.io/project-codeflare/codeflare-self-test:${VERSION-latest}",
+    "build:docker:self-test": "npm run build:headless-maybe && docker build -f deploy/self-test/Dockerfile -t ${SELF_TEST_IMAGE-ghcr.io/project-codeflare/codeflare-self-test:${VERSION-latest}} .",
+    "run:self-test": "./deploy/self-test/test-in-kind.sh",
     "build:headless-maybe": "if [ ! -n \"$FAST\" ]; then npm run build:headless; fi",
     "build:headless": "CLIENT_HOME=$PWD webpack-cli --mode=${MODE-production} --config node_modules/@kui-shell/webpack/headless-webpack.config.js",
     "docker:debug": "ENTRYPOINT=bash ./bin/codeflare -d",
@@ -97,9 +99,11 @@
         "rm -rf ./dist/headless",
         "npm run build:electron:all",
         "VERSION=$version npm run release:cask",
+        "VERSION=$version npm run build:docker:self-test",
         "VERSION=$version npm run build:docker:log-aggregator"
       ],
       "after:release": [
+        "VERSION=$version npm run publish:docker:self-test",
         "VERSION=$version npm run publish:docker:log-aggregator"
       ]
     },
diff --git a/tests/kind/run.sh b/tests/kind/run.sh
diff --git a/tests/kind/values.sh b/tests/kind/values.sh
