Install monitoring-satellite with obs-installer

.werft/observability/install-satellite.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_PATH=$(realpath "$(dirname "$0")")
+
+if [[ -z "${PREVIEW_NAME}" ]]; then
+  echo "Must set PREVIEW_NAME variable" 1>&2
+  exit 1
+fi
+
+if [[ -z "${KUBE_PATH}" ]]; then
+  echo "Must set KUBE_PATH variable" 1>&2
+  exit 1
+fi
+
+# exports all vars
+shopt -os allexport
+
+kubectl --kubeconfig "${KUBE_PATH}" create ns monitoring-satellite || true
+kubectl --kubeconfig "${KUBE_PATH}" create ns certmanager || true
+
+if ! command -v envsubst; then
+  go install github.com/a8m/envsubst/cmd/envsubst@latest
+fi
+
+obsDir="${SCRIPT_PATH}/observability"
+mkdir -p "${obsDir}"
+git clone https://roboquat:"$(cat /mnt/secrets/monitoring-satellite-preview-token/token)"@github.com/gitpod-io/observability.git "${obsDir}"
+cd "${obsDir}/installer"
+
+tmpdir=$(mktemp -d)
+
+envsubst <"${SCRIPT_PATH}/manifests/monitoring-satellite.yaml" | go run main.go render --output-split-files "${tmpdir}" --config -
+
+pushd "${tmpdir}"
+
+# we have to apply the CRDs first and wait until they are available before we can apply the rest
+find . -name "*CustomResourceDefinition*" -exec kubectl --kubeconfig "${KUBE_PATH}" apply -f {} --server-side \;
+
+# wait for the CRDs
+kubectl --kubeconfig "${KUBE_PATH}" -n monitoring-satellite wait --for condition=established --timeout=60s crd/servicemonitors.monitoring.coreos.com
+
+kubectl --kubeconfig "${KUBE_PATH}" apply --server-side -f .
+
+kubectl --kubeconfig "${KUBE_PATH}" patch deployments.apps -n monitoring-satellite grafana --type=json -p="[{'op': 'remove', 'path': '/spec/template/spec/nodeSelector'}]"
+
+popd

.werft/observability/manifests/monitoring-satellite.yaml

@@ -0,0 +1,46 @@
+namespace: monitoring-satellite
+tracing:
+  install: true
+  honeycombAPIKey: ${HONEYCOMB_API_KEY}
+  honeycombDataset: preview-environments
+certmanager:
+  installServiceMonitors: true
+pyrra:
+  install: true
+prometheus:
+  externalLabels:
+    cluster: ${PREVIEW_NAME}
+    environment: preview-environments
+  resources:
+    requests:
+      cpu: 50m
+      memory: 200Mi
+  remoteWrite:
+    - username: ${PROM_REMOTE_WRITE_USER}
+      password: ${PROM_REMOTE_WRITE_PASSWORD}
+      url: "https://victoriametrics.gitpod.io/api/v1/write"
+      writeRelabelConfigs:
+          - action: keep
+            regex: "rest_client_requests_total.*|http_prober_.*"
+            separator: ";"
+            sourceLabels:
+            - __name__
+            - job
+imports:
+  yaml:
+    - gitURL: https://github.com/gitpod-io/observability
+      path: monitoring-satellite/manifests/kube-prometheus-rules
+    - gitURL: https://github.com/gitpod-io/observability
+      path: monitoring-satellite/manifests/kubescape
+    - gitURL: https://github.com/gitpod-io/observability
+      path: monitoring-satellite/manifests/grafana
+    - gitURL: https://github.com/gitpod-io/observability
+      path: monitoring-satellite/manifests/probers
+    - gitURL: https://github.com/gitpod-io/gitpod
+      path: operations/observability/mixins/workspace/rules
+    - gitURL: https://github.com/gitpod-io/gitpod
+      path: operations/observability/mixins/meta/rules
+    - gitURL: https://github.com/gitpod-io/gitpod
+      path: operations/observability/mixins/IDE/rules
+    - gitURL: https://github.com/gitpod-io/observability
+      path: monitoring-satellite/manifests/crds

.werft/observability/monitoring-satellite.ts

@@ -1,6 +1,5 @@
-import { exec } from "../util/shell";
-import { Werft } from "../util/werft";
-import * as fs from "fs";
+import {exec} from "../util/shell";
+import {Werft} from "../util/werft";
 
 type MonitoringSatelliteInstallerOptions = {
     werft: Werft;
@@ -20,7 +19,8 @@ const sliceName = "observability";
  * Installs monitoring-satellite, while updating its dependencies to the latest commit in the branch it is running.
  */
 export class MonitoringSatelliteInstaller {
-    constructor(private readonly options: MonitoringSatelliteInstallerOptions) {}
+    constructor(private readonly options: MonitoringSatelliteInstallerOptions) {
+    }
 
     public async install() {
         const {
@@ -31,112 +31,16 @@ export class MonitoringSatelliteInstaller {
 
         werft.log(
             sliceName,
-            `Cloning observability repository - Branch: ${branch}`,
-        );
-        exec(
-            `git clone --branch ${branch} https://roboquat:$(cat /mnt/secrets/monitoring-satellite-preview-token/token)@github.com/gitpod-io/observability.git`,
-            { silent: true },
-        );
-        let currentCommit = exec(`git rev-parse HEAD`, { silent: true }).stdout.trim();
-        let pwd = exec(`pwd`, { silent: true }).stdout.trim();
-        werft.log(
-            sliceName,
-            `Updating Gitpod's mixin in monitoring-satellite's jsonnetfile.json to latest commit SHA: ${currentCommit}`,
+            `Installing observability stack - Branch: ${branch}`,
         );
 
-        let jsonnetFile = JSON.parse(fs.readFileSync(`${pwd}/observability/jsonnetfile.json`, "utf8"));
-        jsonnetFile.dependencies.forEach((dep) => {
-            if (dep.name == "gitpod") {
-                dep.version = currentCommit;
-            }
-        });
-        fs.writeFileSync(`${pwd}/observability/jsonnetfile.json`, JSON.stringify(jsonnetFile));
-        exec(`cd observability && jb update`, { slice: sliceName });
-
-            // As YAML is indentation sensitive we're using json instead so we don't have to worry about
-            // getting the indentation right when formatting the code in TypeScript.
-            const observabilityInstallerRenderCmd = `cd observability && \
-            make generate && \
-            ./hack/deploy-crds.sh --kubeconfig ${this.options.kubeconfigPath} && \
-            kubectl create ns monitoring-satellite --kubeconfig ${this.options.kubeconfigPath} || true && \
-            cd installer && echo '
-            {
-                "gitpod": {
-                    "installServiceMonitors": true
-                },
-                "pyrra": {
-                    "install": true
-                },
-                "tracing": {
-                    "install": true,
-                    "honeycombAPIKey": "${process.env.HONEYCOMB_API_KEY}",
-                    "honeycombDataset": "preview-environments",
-                },
-                "prometheus": {
-                    "externalLabels": {
-                        "cluster": "${previewName}",
-                        "environment": "preview-environments",
-                    },
-                    "resources": {
-                        "requests": {
-                            "memory": "200Mi",
-                            "cpu": "50m",
-                        },
-                    },
-                    "remoteWrite": [{
-                        "username": "${process.env.PROM_REMOTE_WRITE_USER}",
-                        "password": "${process.env.PROM_REMOTE_WRITE_PASSWORD}",
-                        "url": "https://victoriametrics.gitpod.io/api/v1/write",
-                        "writeRelabelConfigs": [{
-                            "sourceLabels": ["__name__", "job"],
-                            "separator": ";",
-                            "regex": "rest_client_requests_total.*|http_prober_.*",
-                            "action": "keep",
-                        }],
-                    }],
-                },
-                "imports": {
-                    "yaml": [{
-                            "gitURL": "https://github.com/gitpod-io/observability",
-                            "path": "monitoring-satellite/manifests/kube-prometheus-rules",
-                        },
-                        {
-                            "gitURL": "https://github.com/gitpod-io/observability",
-                            "path": "monitoring-satellite/manifests/kubescape",
-                        },
-                        {
-                            "gitURL": "https://github.com/gitpod-io/observability",
-                            "path": "monitoring-satellite/manifests/grafana",
-                        },
-                        {
-                            "gitURL": "https://github.com/gitpod-io/observability",
-                            "path": "monitoring-satellite/manifests/probers",
-                        },
-                        {
-                            "gitURL": "https://github.com/gitpod-io/gitpod",
-                            "path": "operations/observability/mixins/workspace/rules",
-                        },
-                        {
-                            "gitURL": "https://github.com/gitpod-io/gitpod",
-                            "path": "operations/observability/mixins/meta/rules",
-                        },
-                        {
-                            "gitURL": "https://github.com/gitpod-io/gitpod",
-                            "path": "operations/observability/mixins/IDE/rules",
-                        },
-                    ],
-                },
-            }' | go run main.go render --config - | kubectl --kubeconfig ${this.options.kubeconfigPath} apply -f -`;
-            const renderingResult = exec(observabilityInstallerRenderCmd, { silent: false, dontCheckRc: true});
-            if (renderingResult.code > 0) {
-                const err = new Error(`Failed rendering YAML with exit code ${renderingResult.code}`)
-                renderingResult.stderr.split('\n').forEach(stderrLine => werft.log(sliceName, stderrLine))
-                werft.failSlice(sliceName, err)
-                return
-            }
+        const installSatellite = exec(`KUBE_PATH=${this.options.kubeconfigPath} PREVIEW_NAME=${previewName} .werft/observability/install-satellite.sh`, {slice: sliceName});
 
-            // The grafana YAML files we're importing have nodeSelector tied to a nodepool that don't exist in previews
-            // We're hot-patching the removal os such nodeSelector to make sure Grafana starts
-            exec(`kubectl patch deployments.apps -n monitoring-satellite grafana --type=json -p="[{'op': 'remove', 'path': '/spec/template/spec/nodeSelector'}]"`)
+        if (installSatellite.code > 0) {
+            const err = new Error(`Failed installing monitoring-satellite`)
+            installSatellite.stderr.split('\n').forEach(stderrLine => werft.log(sliceName, stderrLine))
+            werft.failSlice(sliceName, err)
+            return
+        }
     }
 }