Add in-cluster builds (#11)

Add in-cluster builds.
Improve watchers and predicates.

Author: qbarrand

.github/workflows/e2e.yaml

@@ -82,6 +82,7 @@ jobs:
         run: |
           wget https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
           sudo dpkg -i ./minikube_latest_amd64.deb
+
       - name: Start minikube and wait until CoreDNS is available
         run: |
           minikube start --driver=docker
@@ -131,6 +132,7 @@ jobs:
             echo "Unexpected lsmod output - the module should not be loaded"
             exit 1
           fi
+
       - name: Add an ooto-ci Module that contains a valid mapping
         run: |
           sed -e "s/KVER_CHANGEME/$(uname -r)/g" \
@@ -139,6 +141,7 @@ jobs:
             ci/module-ooto-ci.template.yaml | tee module-ooto-ci.yaml
 
           kubectl apply -f module-ooto-ci.yaml
+
       - name: Check that the module gets loaded on the node
         run: |
           until minikube ssh -- lsmod | grep ooto_ci_a; do
@@ -199,6 +202,7 @@ jobs:
         run: |
           wget https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
           sudo dpkg -i ./minikube_latest_amd64.deb
+
       - name: Start minikube and wait until CoreDNS is available
         run: |
           minikube start --driver=docker -n 2
@@ -332,3 +336,138 @@ jobs:
       - name: Get all operator logs
         run: kubectl logs deployment.apps/oot-operator-controller-manager -n oot-operator-system
         if: ${{ always() }}
+
+  in-cluster-build:
+    runs-on: ubuntu-latest
+
+    name: In-cluster build
+
+    needs: [build-operator-image, build-drivercontainer-image]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Download and install minikube
+        run: |
+          wget https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
+          sudo dpkg -i ./minikube_latest_amd64.deb
+
+      - name: Start minikube and wait until CoreDNS is available
+        run: |
+          minikube start --driver=docker --addons registry,registry-aliases
+          kubectl wait --for=condition=available deployment coredns -n kube-system
+
+          kubectl apply -f ci/registry-nodeport.yaml
+
+      - name: Install skopeo
+        run: |
+          sudo apt -y update
+          sudo apt -y install skopeo
+
+      - name: Download container images
+        uses: actions/download-artifact@v3
+        with:
+          name: ci-images
+
+      - name: Import DriverContainer base into the internal-registry
+        run: |
+          MINIKUBE_REGISTRY_EXT="$(minikube service registry-nodeport -n kube-system --format '{{.IP}}:{{.Port}}' --url)"
+          skopeo copy --dest-tls-verify=false docker-archive:ooto-kmod_local.tar docker://${MINIKUBE_REGISTRY_EXT}/ooto-base:local
+
+      - name: Import the OOTO image into minikube
+        run: minikube image load ooto_local.tar
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.6
+
+      - name: Deploy OOTO
+        run: make deploy
+        env:
+          KUSTOMIZE_CONFIG_DEFAULT: ci/install-ci
+
+      - name: Wait until the OOTO Deployment is Available
+        run: kubectl wait --for condition=Available deployments.apps -n oot-operator-system oot-operator-controller-manager
+        timeout-minutes: 1
+
+      - name: Describe the Deployment / pods and get their YAML if that failed
+        run: |
+          kubectl describe deployments.apps -n oot-operator-system oot-operator-controller-manager
+          kubectl get -o yaml deployments.apps -n oot-operator-system oot-operator-controller-manager
+
+          kubectl describe pod -n oot-operator-system
+          kubectl get -o yaml pod -n oot-operator-system
+        if: ${{ failure() }}
+
+      - name: Add an ooto-ci Module that contains a valid mapping
+        run: |
+          sed -e "s/KVER_CHANGEME/$(uname -r)/g" ci/module-ooto-ci-build.template.yaml | tee module-ooto-ci.yaml
+
+          kubectl apply -f module-ooto-ci.yaml
+
+      - name: Wait for the job to be created
+        run: |
+          until kubectl get job -l oot.node.kubernetes.io/module.name | grep ooto; do
+            sleep 3
+          done
+        timeout-minutes: 1
+
+      - name: Wait for Job completion
+        run: kubectl wait --for condition=complete job -l oot.node.kubernetes.io/module.name --timeout=-1s
+        timeout-minutes: 2
+
+      - name: Collect job logs
+        run: |
+          JOB_NAME=$(kubectl get jobs.batch -l oot.node.kubernetes.io/module.name --template='{{ (index .items 0).metadata.name }}')
+          kubectl logs jobs.batch/${JOB_NAME}
+        if: ${{ always() }}
+
+      - name: Check that the module gets loaded on the node
+        run: |
+          until minikube ssh -- lsmod | grep ooto_ci_a; do
+            sleep 3
+          done
+        timeout-minutes: 1
+
+      - name: Remove the Module
+        run: kubectl delete -f module-ooto-ci.yaml
+
+      - name: Check that the module gets unloaded from the node
+        run: |
+          until ! minikube ssh -- lsmod | grep ooto_ci_a; do
+            sleep 3
+          done
+        timeout-minutes: 1
+
+      - name: Get all resources in the oot-operator-system namespace
+        run: kubectl get all -n oot-operator-system
+        if: ${{ always() }}
+
+      - name: Get all resources in the default namespace
+        run: kubectl get all
+        if: ${{ always() }}
+
+      - name: Describe nodes
+        run: kubectl describe node
+        if: ${{ always() }}
+
+      - name: Describe Jobs
+        run: kubectl describe job
+        if: ${{ always() }}
+
+      - name: Describe DaemonSets
+        run: kubectl describe daemonset -l oot.node.kubernetes.io/module.name
+        if: ${{ always() }}
+
+      - name: Describe Pods
+        run: kubectl describe pod -l oot.node.kubernetes.io/module.name
+        if: ${{ always() }}
+
+      - name: Collect dmesg
+        run: sudo dmesg
+        if: ${{ always() }}
+
+      - name: Get all operator logs
+        run: kubectl logs deployment.apps/oot-operator-controller-manager -n oot-operator-system
+        if: ${{ always() }}

api/v1beta1/module_types.go

@@ -21,9 +21,42 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type PullOptions struct {
+	// +optional
+
+	// When Insecure is true, images can be pulled from an insecure (plain HTTP) registry.
+	Insecure bool `json:"insecure"`
+}
+
+type PushOptions struct {
+	// +optional
+
+	// When Insecure is true, built images can be pushed to an insecure (plain HTTP) registry.
+	Insecure bool `json:"insecure"`
+}
+
+type Build struct {
+	Dockerfile string `json:"dockerfile"`
+
+	// +optional
+
+	// Pull contains settings determining how to check if the DriverContainer image already exists.
+	Pull PullOptions `json:"pull"`
+
+	// +optional
+
+	// Push contains settings determining how to push a built DriverContainer image.
+	Push PushOptions `json:"push"`
+}
+
 // KernelMapping pairs kernel versions with a DriverContainer image.
 // Kernel versions can be matched literally or using a regular expression.
 type KernelMapping struct {
+	// +optional
+
+	// Build enables in-cluster builds for this mapping and allows overriding the Module's build settings.
+	Build *Build `json:"build"`
+
 	// ContainerImage is the name of the DriverContainer image that should be used to deploy the module.
 	ContainerImage string `json:"containerImage"`
 
@@ -40,6 +73,9 @@ type KernelMapping struct {
 
 // ModuleSpec describes how the OOT operator should deploy a Module on those nodes that need it.
 type ModuleSpec struct {
+	// +optional
+	Build Build `json:"build"`
+
 	// +optional
 
 	// AdditionalVolumes is a list of volumes that will be attached to the DriverContainer / DevicePlugin pod,

api/v1beta1/zz_generated.deepcopy.go

@@ -0,0 +1,30 @@
+---
+apiVersion: ooto.sigs.k8s.io/v1beta1
+kind: Module
+metadata:
+  name: ooto-ci-build
+spec:
+  driverContainer:
+    name: overwritten-anyway
+    command: [sleep, infinity]
+    lifecycle:
+      postStart:
+        exec:
+          command: [modprobe, -vd, /opt, ooto_ci_a]
+      preStop:
+        exec:
+          command: [modprobe, -rvd, /opt, ooto_ci_a]
+    securityContext:
+      capabilities:
+        add: [SYS_MODULE]
+  kernelMappings:
+    - literal: KVER_CHANGEME
+      containerImage: registry.minikube/ooto-kmod:local
+      build:
+        pull:
+          insecure: true
+        push:
+          insecure: true
+        dockerfile: FROM registry.minikube/ooto-base:local
+  selector:
+    kubernetes.io/hostname: minikube

ci/module-ooto-ci-build.template.yaml

@@ -11,10 +11,10 @@ spec:
     lifecycle:
       postStart:
         exec:
-          command: [modprobe, -d, /opt, -v, KMOD_CHANGEME]
+          command: [modprobe, -vd, /opt, KMOD_CHANGEME]
       preStop:
         exec:
-          command: [modprobe, -d, /opt, -rv, KMOD_CHANGEME]
+          command: [modprobe, -rvd, /opt, KMOD_CHANGEME]
     securityContext:
       capabilities:
         add: [SYS_MODULE]

ci/module-ooto-ci.template.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: registry-nodeport
+  namespace: kube-system
+spec:
+  selector:
+    actual-registry: 'true'
+    kubernetes.io/minikube-addons: registry
+  ports:
+    - port: 5000
+      targetPort: 5000
+  type: NodePort