Merge pull request #2006 from consideRatio/pr/file-injection

Allow extraFiles to be injected to hub / singleuser pods and automatically load config in /usr/local/etc/jupyterhub_config.d

Author: minrk

dev-config.yaml

@@ -48,8 +48,59 @@ hub:
     requests:
       memory: 0
       cpu: 0
+  extraFiles:
+    my_config:
+      mountPath: /usr/local/etc/jupyterhub/jupyterhub_config.d/my_config.py
+      stringData: |
+        with open("/tmp/created-by-extra-files-config.txt", "w") as f:
+            f.write("hello world!")
+    binaryData1: &binaryData1
+      mountPath: /tmp/binaryData.txt
+      mode: 0666
+      binaryData: |
+        aGVsbG8gd
+        29ybGQhCg==
+    binaryData2: &binaryData2
+      mountPath: /tmp/dir1/binaryData.txt
+      mode: 0666
+      binaryData: aGVsbG8gd29ybGQhCg==
+    stringData1: &stringData1
+      mountPath: /tmp/stringData.txt
+      mode: 0666
+      stringData: hello world!
+    stringData2: &stringData2
+      mountPath: /tmp/dir1/stringData.txt
+      mode: 0666
+      stringData: hello world!
+    data-yaml: &data-yaml
+      mountPath: /etc/test/data.yaml
+      mode: 0444
+      data:
+        config:
+          map:
+            number: 123
+            string: "hi"
+          list: [1, 2]
+    data-yml: &data-yml
+      <<: *data-yaml
+      mountPath: /etc/test/data.yml
+    data-json: &data-json
+      <<: *data-yaml
+      mountPath: /etc/test/data.json
+    data-toml: &data-toml
+      <<: *data-yaml
+      mountPath: /etc/test/data.toml
 
 singleuser:
+  extraFiles:
+    binaryData1: *binaryData1
+    binaryData2: *binaryData2
+    stringData1: *stringData1
+    stringData2: *stringData2
+    data-yaml: *data-yaml
+    data-yml: *data-yml
+    data-json: *data-json
+    data-toml: *data-toml
   storage:
     type: none
   memory:

images/hub/Dockerfile

@@ -64,4 +64,4 @@ ARG PIP_TOOLS=
 RUN test -z "$PIP_TOOLS" || pip install --no-cache pip-tools==$PIP_TOOLS
 
 USER ${NB_USER}
-CMD ["jupyterhub", "--config", "/etc/jupyterhub/jupyterhub_config.py"]
+CMD ["jupyterhub", "--config", "/usr/local/etc/jupyterhub/jupyterhub_config.py"]

jupyterhub/files/hub/jupyterhub_config.py

@@ -235,7 +235,6 @@ def camelCaseify(s):
 
 # Configure dynamically provisioning pvc
 storage_type = get_config("singleuser.storage.type")
-
 if storage_type == "dynamic":
     pvc_name_template = get_config("singleuser.storage.dynamic.pvcNameTemplate")
     c.KubeSpawner.pvc_name_template = pvc_name_template
@@ -280,6 +279,43 @@ def camelCaseify(s):
         }
     ]
 
+# Inject singleuser.extraFiles as volumes and volumeMounts with data loaded from
+# the dedicated k8s Secret prepared to hold the extraFiles actual content.
+extra_files = get_config("singleuser.extraFiles", {})
+if extra_files:
+    volume = {
+        "name": "files",
+    }
+    items = []
+    for file_key, file_details in extra_files.items():
+        # Each item is a mapping of a key in the k8s Secret to a path in this
+        # abstract volume, the goal is to enable us to set the mode /
+        # permissions only though so we don't change the mapping.
+        item = {
+            "key": file_key,
+            "path": file_key,
+        }
+        if "mode" in file_details:
+            item["mode"] = file_details["mode"]
+        items.append(item)
+    volume["secret"] = {
+        "secretName": get_name("singleuser"),
+        "items": items,
+    }
+    c.KubeSpawner.volumes.append(volume)
+
+    volume_mounts = []
+    for file_key, file_details in extra_files.items():
+        volume_mounts.append(
+            {
+                "mountPath": file_details["mountPath"],
+                "subPath": file_key,
+                "name": "files",
+            }
+        )
+    c.KubeSpawner.volume_mounts.extend(volume_mounts)
+
+# Inject extraVolumes / extraVolumeMounts
 c.KubeSpawner.volumes.extend(get_config("singleuser.storage.extraVolumes", []))
 c.KubeSpawner.volume_mounts.extend(
     get_config("singleuser.storage.extraVolumeMounts", [])
@@ -371,6 +407,15 @@ def camelCaseify(s):
     c.JupyterHub.log_level = "DEBUG"
     c.Spawner.debug = True
 
+# load /usr/local/etc/jupyterhub/jupyterhub_config.d config files
+config_dir = "/usr/local/etc/jupyterhub/jupyterhub_config.d"
+if os.path.isdir(config_dir):
+    for file_name in sorted(os.listdir(config_dir)):
+        print(f"Loading {config_dir} config: {file_name}")
+        with open(f"{config_dir}/{file_name}") as f:
+            file_content = f.read()
+        # compiling makes debugging easier: https://stackoverflow.com/a/437857
+        exec(compile(source=file_content, filename=file_name, mode="exec"))
 
 # load potentially seeded secrets
 c.JupyterHub.proxy_auth_token = get_secret_value("JupyterHub.proxy_auth_token")

jupyterhub/files/hub/z2jh.py

@@ -15,7 +15,7 @@ def _load_config():
     """Load the Helm chart configuration used to render the Helm templates of
     the chart from a mounted k8s Secret."""
 
-    path = f"/etc/jupyterhub/secret/values.yaml"
+    path = f"/usr/local/etc/jupyterhub/secret/values.yaml"
     if os.path.exists(path):
         print(f"Loading {path}")
         with open(path) as f:
@@ -28,7 +28,7 @@ def _load_config():
 def _get_config_value(key):
     """Load value from the k8s ConfigMap given a key."""
 
-    path = f"/etc/jupyterhub/config/{key}"
+    path = f"/usr/local/etc/jupyterhub/config/{key}"
     if os.path.exists(path):
         with open(path) as f:
             return f.read()
@@ -40,7 +40,7 @@ def _get_config_value(key):
 def get_secret_value(key):
     """Load value from the k8s Secret given a key."""
 
-    path = f"/etc/jupyterhub/secret/{key}"
+    path = f"/usr/local/etc/jupyterhub/secret/{key}"
     if os.path.exists(path):
         with open(path) as f:
             return f.read()

jupyterhub/schema.yaml

@@ -212,6 +212,135 @@ properties:
           the `--values` or `-f` flag. During merging, lists are replaced while
           dictionaries are updated.
           ```
+      extraFiles: &extraFiles
+        type: string
+        description: |
+          A dictionary with extra files to be injected into the pod's container
+          on startup. This can for example be used to inject: configuration
+          files, custom user interface templates, images, and more.
+
+          ```yaml
+          hub:
+            extraFiles:
+              # The file key is just a reference that doesn't influence the
+              # actual file name.
+              <file key>:
+                # mountPath is required and must be the absolute file path.
+                mountPath: <full file path>
+
+                # Choose one out of the three ways to represent the actual file
+                # content: data, stringData, or binaryData.
+                #
+                # data should be set to a mapping (dictionary). It will in the
+                # end be rendered to either YAML, JSON, or TOML based on the
+                # filename extension that are required to be either .yaml, .yml,
+                # .json, or .toml.
+                #
+                # If your content is YAML, JSON, or TOML, it can make sense to
+                # use data to represent it over stringData as data can be merged
+                # instead of replaced if set partially from separate Helm
+                # configuration files.
+                #
+                # Both stringData and binaryData should be set to a string
+                # representing the content, where binaryData should be the
+                # base64 encoding of the actual file content.
+                #
+                data:
+                  config:
+                    map:
+                      number: 123
+                      string: "hi"
+                    list:
+                      - 1
+                      - 2
+                stringData: |
+                  hello world!
+                binaryData: aGVsbG8gd29ybGQhCg==
+
+                # mode is by default 0644 and you can optionally override it
+                # either by octal notation (example: 0400) or decimal notation
+                # (example: 256).
+                mode: <file system permissions>
+          ```
+
+          **Using --set-file**
+
+          To avoid embedding entire files in the Helm chart configuration, you
+          can use the `--set-file` flag during `helm upgrade` to set the
+          stringData or binaryData field.
+
+          ```yaml
+          hub:
+            extraFiles:
+              my_image:
+                mountPath: /usr/local/share/jupyterhub/static/my_image.png
+
+              # Files in /usr/local/etc/jupyterhub/jupyterhub_config.d are
+              # automatically loaded in alphabetical order of the final file
+              # name when JupyterHub starts.
+              my_config:
+                mountPath: /usr/local/etc/jupyterhub/jupyterhub_config.d/my_jupyterhub_config.py
+          ```
+
+          ```bash
+          # --set-file expects a text based file, so you need to base64 encode
+          # it manually first.
+          base64 my_image.png > my_image.png.b64
+
+          helm upgrade <...> \
+              --set-file hub.extraFiles.my_image.binaryData=./my_image.png.b64 \
+              --set-file hub.extraFiles.my_config.stringData=./my_jupyterhub_config.py
+          ```
+
+          **Common uses**
+
+          1. **JupyterHub template customization**
+
+             You can replace the default JupyterHub user interface templates in
+             the hub pod by injecting new ones to
+             `/usr/local/share/jupyterhub/templates`. These can in turn
+             reference custom images injected to
+             `/usr/local/share/jupyterhub/static`.
+
+          1. **JupyterHub standalone file config**
+
+             Instead of embedding JupyterHub python configuration as a string
+             within a YAML file through
+             [`hub.extraConfig`](schema_hub.extraConfig), you can inject a
+             standalone .py file into
+             `/usr/local/etc/jupyterhub/jupyterhub_config.d` that is
+             automatically loaded.
+
+          1. **Flexible configuration**
+
+             By injecting files, you don't have to embed them in a docker image
+             that you have to rebuild.
+
+             If your configuration file is a YAML/JSON/TOML file, you can also
+             use `data` instead of `stringData` which allow you to set various
+             configuration in separate Helm config files. This can be useful to
+             help dependent charts override only some configuration part of the
+             file, or to allow for the configuration be set through multiple
+             Helm configuration files.
+
+          **Limitations**
+
+          1. File size
+
+             The files in `hub.extraFiles` and `singleuser.extraFiles` are
+             respectively stored in their own k8s Secret resource. As k8s
+             Secret's are limited, typically to 1MB, you will be limited to a
+             total file size of less than 1MB as there is also base64 encoding
+             that takes place reducing available capacity to 75%.
+
+          2. File updates
+
+             The files that are mounted are only set during container startup.
+             This is [because we use
+             `subPath`](https://kubernetes.io/docs/concepts/storage/volumes/#secret)
+             as is required to avoid replacing the content of the entire
+             directory we mount in.
+
       baseUrl:
         type: string
         description: |
@@ -243,10 +372,10 @@ properties:
 
           ```{warning}
           By replacing the entire configuration file, which is mounted to
-          `/etc/jupyterhub/jupyterhub_config.py` by the Helm chart, instead of
-          appending to it with `hub.extraConfig`, you expose your deployment for
-          issues stemming from getting out of sync with the Helm chart's config
-          file.
+          `/usr/local/etc/jupyterhub/jupyterhub_config.py` by the Helm chart,
+          instead of appending to it with `hub.extraConfig`, you expose your
+          deployment for issues stemming from getting out of sync with the Helm
+          chart's config file.
 
           These kind of issues will be significantly harder to debug and
           diagnose, and can due to this could cause a lot of time expenditure
@@ -268,7 +397,7 @@ properties:
             args:
               - "jupyterhub"
               - "--config"
-              - "/etc/jupyterhub/jupyterhub_config.py"
+              - "/usr/local/etc/jupyterhub/jupyterhub_config.py"
               - "--debug"
               - "--upgrade-db"
           ```
@@ -1154,6 +1283,7 @@ properties:
         description: |
           Deprecated and no longer does anything. Use the user-scheduler instead
           in order to accomplish a good packing of the user pods.
+      extraFiles: *extraFiles
       extraEnv:
         type: object
         description: |