Make kubelet dir configurable (fixes #229) (#232)

# Description

This should fix #229 from secret-operator's perspective. There are still outstanding issues in zk-op that I'll open a new issue for.

Author: nightkr

CHANGELOG.md

@@ -11,9 +11,12 @@ All notable changes to this project will be documented in this file.
   - After upgrading you will need to
     `rmdir /var/lib/kubelet/plugins_registry/secrets.stackable.tech-reg.sock` manually.
     This applies to *all* users, not just Microk8s.
+- Made kubeletDir configurable ([#232]).
+  - Microk8s users will need to `--set kubeletDir=/var/snap/microk8s/common/var/lib/kubelet`.
 
 [#212]: https://github.com/stackabletech/secret-operator/pull/212
 [#231]: https://github.com/stackabletech/secret-operator/pull/231
+[#232]: https://github.com/stackabletech/secret-operator/pull/232
 
 ## [0.6.0] - 2022-11-07
 

Tiltfile

@@ -2,7 +2,7 @@ default_registry("docker.stackable.tech/sandbox")
 
 custom_build(
     'docker.stackable.tech/sandbox/secret-operator',
-    'nix run -f . crate2nix generate && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/secret-operator" && ./result/load-image | docker load',
+    'nix shell -f . crate2nix -c crate2nix generate && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/secret-operator" && ./result/load-image | docker load',
     deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'],
     # ignore=['result*', 'Cargo.nix', 'target', *.yaml],
     outputs_image_ref_to='result/ref',

default.nix

@@ -53,6 +53,7 @@ rec {
     }
   ];
 
-  crate2nix = pkgs.crate2nix;
+  # need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264
+  crate2nix = pkgs.callPackage sources.crate2nix {};
   tilt = pkgs.tilt;
 }

deploy/helm/secret-operator/templates/daemonset.yaml

@@ -45,7 +45,7 @@ spec:
             - name: csi
               mountPath: /csi
             - name: mountpoint
-              mountPath: /var/lib/kubelet/pods
+              mountPath: {{ .Values.kubeletDir }}/pods
               mountPropagation: Bidirectional
         - name: external-provisioner
           image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
@@ -60,7 +60,7 @@ spec:
           image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
           args:
             - --csi-address=/csi/csi.sock
-            - --kubelet-registration-path=/var/lib/kubelet/plugins/secrets.stackable.tech/csi.sock
+            - --kubelet-registration-path={{ .Values.kubeletDir }}/plugins/secrets.stackable.tech/csi.sock
           volumeMounts:
             - name: registration-sock
               mountPath: /registration
@@ -71,13 +71,13 @@ spec:
           hostPath:
             # node-driver-registrar appends a driver-unique filename to this path to avoid conflicts
             # see https://github.com/stackabletech/secret-operator/issues/229 for why this path should not be too long
-            path: /var/lib/kubelet/plugins_registry
+            path: {{ .Values.kubeletDir }}/plugins_registry
         - name: csi
           hostPath:
-            path: /var/lib/kubelet/plugins/secrets.stackable.tech/
+            path: {{ .Values.kubeletDir }}/plugins/secrets.stackable.tech/
         - name: mountpoint
           hostPath:
-            path: /var/lib/kubelet/pods/
+            path: {{ .Values.kubeletDir }}/pods/
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/helm/secret-operator/values.yaml

@@ -50,3 +50,6 @@ nodeSelector: {}
 tolerations: []
 
 affinity: {}
+
+# Kubelet dir may vary in environments such as microk8s, see https://github.com/stackabletech/secret-operator/issues/229
+kubeletDir: /var/lib/kubelet

docs/modules/ROOT/pages/installation.adoc

@@ -36,6 +36,11 @@ $ helm install secret-operator stackable/secret-operator
 
 Helm will deploy the operator in Kubernetes containers and apply the CRDs. You're now ready to deploy secrets!
 
+=== Microk8s
+
+Microk8s uses a non-standard Kubelet state directory. Installing secret-operator on Microk8s requires the argument
+`--set kubeletDir=/var/snap/microk8s/common/var/lib/kubelet` to be added to the `helm install` command.
+
 == Building the operator from source
 
 See xref:building.adoc[] for more information.

nix/sources.json

@@ -1,14 +1,26 @@
 {
+    "crate2nix": {
+        "branch": "master",
+        "description": "nix build file generator for rust crates",
+        "homepage": "",
+        "owner": "kolloch",
+        "repo": "crate2nix",
+        "rev": "fd2b9cf70ef55529b6793036b648ce2d9e2e7ba1",
+        "sha256": "0cc15msc0c3asmilr1czsjf4vbzqzj13w3ff33nyq6gxq3md84k3",
+        "type": "tarball",
+        "url": "https://github.com/kolloch/crate2nix/archive/fd2b9cf70ef55529b6793036b648ce2d9e2e7ba1.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "nixpkgs": {
         "branch": "nixpkgs-unstable",
         "description": "Nix Packages collection",
         "homepage": "",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3bbb296d9a0088c314ce83038b896753bbe33acb",
-        "sha256": "0dlq1zjcki30k2afg749a7c34vyb86sx1irab48l2g214nhj83hw",
+        "rev": "79feedf38536de2a27d13fe2eaf200a9c05193ba",
+        "sha256": "0ndv4nq0rap8m0qr50qliz6fib5gd8bg5flr67lmvcifx6lmri5v",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/3bbb296d9a0088c314ce83038b896753bbe33acb.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/79feedf38536de2a27d13fe2eaf200a9c05193ba.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }