WIP: Reword later

mads-hartmann · mads-hartmann · commit 060378057ec2 · 2022-05-03T07:59:36.000Z
diff --git a/.werft/platform-delete-preview-environments-cron.ts b/.werft/platform-delete-preview-environments-cron.ts
@@ -93,7 +93,7 @@ class HarvesterPreviewEnvironment {
      */
     static expectedNamespaceFromBranch(branch: string): string {
         const previewName = previewNameFromBranchName(branch)
-        return `${HarvesterPreviewEnvironment.namespacePrefix}-${previewName}`
+        return `${HarvesterPreviewEnvironment.namespacePrefix}${previewName}`
     }
 }
 
@@ -113,7 +113,7 @@ class CoreDevPreviewEnvironment {
     }
 
     async delete(sliceID: string): Promise<void> {
-        await wipePreviewEnvironmentAndNamespace(helmInstallName, this.name, CORE_DEV_KUBECONFIG_PATH, { slice: sliceID })
+        await wipePreviewEnvironmentAndNamespace(helmInstallName, this.namespace, CORE_DEV_KUBECONFIG_PATH, { slice: sliceID })
     }
 
     async removeDNSRecords(sliceID: string) {
@@ -191,7 +191,7 @@ class CoreDevPreviewEnvironment {
      */
     static expectedNamespaceFromBranch(branch: string): string {
         const previewName = previewNameFromBranchName(branch)
-        return `${CoreDevPreviewEnvironment.namespacePrefix}-${previewName}`
+        return `${CoreDevPreviewEnvironment.namespacePrefix}${previewName}`
     }
 
 }
@@ -251,6 +251,53 @@ async function deletePreviewEnvironments() {
 
     werft.phase("Determining which preview environments are stale");
 
+    const previewsToDelete = await determineStalePreviewEnvironments({
+        branches: branches,
+        previews: previews
+    })
+
+    if (previewsToDelete.length == 0) {
+        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, "No stale preview environments.")
+        werft.done(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS)
+        return
+    } else {
+        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Found ${previewsToDelete.length} stale preview environments`)
+        werft.done(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS)
+    }
+
+    werft.phase("Deleting stale preview environments")
+    if (DRY_RUN) {
+        previewsToDelete.forEach(preview => {
+            werft.log(SLICES.DELETING_PREVIEW_ENVIRONMNETS, `Would have deleted preview environment ${preview.name} (${preview.namespace})`)
+        })
+        werft.done(SLICES.DELETING_PREVIEW_ENVIRONMNETS)
+        return
+    }
+
+    try {
+        const promises: Promise<any>[] = [];
+        previewsToDelete.forEach(preview => promises.push(removePreviewEnvironment(preview)))
+        await Promise.all(promises)
+        werft.done(SLICES.DELETING_PREVIEW_ENVIRONMNETS)
+    } catch (err) {
+        werft.fail(SLICES.DELETING_PREVIEW_ENVIRONMNETS, err)
+    }
+}
+
+/**
+ * Determines if preview environemnts are stale and should be deleted
+ *
+ * As we don't have a mapping from preview environnment -> Git branch we have to
+ * go about this in a bit of a backwards way.
+ *
+ * Based on the active git branches we compute a set of "expected" preview environments
+ * and then use that to compare with the "live" preview environemnts to decide which
+ * ones to keep
+ */
+async function determineStalePreviewEnvironments(options: {previews: PreviewEnvironment[], branches: string[]}): Promise<PreviewEnvironment[]> {
+
+    const {branches, previews} = options
+
     // The set of namespaces that we would expect based on the open branches.
     // This contains both the core-dev and the harvester namespaces as we only use this set for
     // testing membership in situations where we don't care if the preview environment is based on
@@ -276,48 +323,27 @@ async function deletePreviewEnvironments() {
             HarvesterPreviewEnvironment.expectedNamespaceFromBranch(branch)
         ]))
 
-    const deleteDueToMissingBranch = previews.filter(preview => {
-        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to missing branch`)
-        return !previewNamespaceBasedOnBranches.has(preview.namespace)
-    })
-
-    const deleteDueToNoCommitActivity = previews.filter(preview => {
-        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to no recent commit activity`)
-        return previewNamespaceBasedOnStaleBranches.has(preview.namespace)
-    })
-
-    const deleteDueToNoDBActivity = previews.filter((preview: PreviewEnvironment) => {
-        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to no recent DB activity`)
-        return preview.isInactive()
-    })
+    const previewsToDelete = previews.filter((preview: PreviewEnvironment) => {
+        if (!previewNamespaceBasedOnBranches.has(preview.namespace)) {
+            werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to missing branch`)
+            return true
+        }
 
-    const previewsToDelete = new Set([...deleteDueToMissingBranch, ...deleteDueToNoCommitActivity, ...deleteDueToNoDBActivity])
+        if (previewNamespaceBasedOnStaleBranches.has(preview.namespace)) {
+            werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to no recent commit activity`)
+            return true
+        }
 
-    if (previewsToDelete.size == 0) {
-        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, "No stale preview environments.")
-        werft.done(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS)
-        return
-    } else {
-        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Found ${previewsToDelete.size} stale preview environments`)
-    }
+        if (preview.isInactive()) {
+            werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to no recent DB activity`)
+            return true
+        }
 
-    werft.phase("Deleting stale preview environments")
-    if (DRY_RUN) {
-        previewsToDelete.forEach(preview => {
-            werft.log(SLICES.DELETING_PREVIEW_ENVIRONMNETS, `Would have deleted preview environment ${preview.name} (${preview.namespace})`)
-        })
-        werft.done(SLICES.DELETING_PREVIEW_ENVIRONMNETS)
-        return
-    }
+        werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) active`)
+        return false
+    })
 
-    try {
-        const promises: Promise<any>[] = [];
-        previewsToDelete.forEach(preview => promises.push(removePreviewEnvironment(preview)))
-        await Promise.all(promises)
-        werft.done(SLICES.DELETING_PREVIEW_ENVIRONMNETS)
-    } catch (err) {
-        werft.fail(SLICES.DELETING_PREVIEW_ENVIRONMNETS, err)
-    }
+    return previewsToDelete
 }
 
 async function removePreviewEnvironment(previewEnvironment: PreviewEnvironment) {
@@ -330,12 +356,12 @@ async function removePreviewEnvironment(previewEnvironment: PreviewEnvironment)
         await previewEnvironment.delete(sliceID)
         werft.done(sliceID)
     } catch (e) {
-        werft.fail(sliceID, e)
+        werft.failSlice(sliceID, e)
     }
 }
 
 async function removeCertificate(preview: string, kubectlConfig: string, slice: string) {
-    return exec(`kubectl --kubeconfig ${kubectlConfig} -n certs delete cert ${preview}`, {slice: slice, async: true})
+    return exec(`kubectl --kubeconfig ${kubectlConfig} -n certs delete --ignore-not-found=true cert ${preview}`, {slice: slice, async: true})
 }
 
 async function cleanLoadbalancer() {
diff --git a/.werft/platform-delete-preview-environments-cron.yaml b/.werft/platform-delete-preview-environments-cron.yaml
@@ -19,6 +19,9 @@ pod:
   - name: harvester-kubeconfig
     secret:
       secretName: harvester-kubeconfig
+  - name: harvester-k3s-dockerhub-pull-account
+    secret:
+      secretName: harvester-k3s-dockerhub-pull-account
   containers:
   - name: build
     image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go.4
@@ -33,6 +36,8 @@ pod:
       readOnly: true
     - name: harvester-kubeconfig
       mountPath: /mnt/secrets/harvester-kubeconfig
+    - name: harvester-k3s-dockerhub-pull-account
+      mountPath: /mnt/secrets/harvester-k3s-dockerhub-pull-account
     env:
     - name: WERFT_HOST
       value: "werft.werft.svc.cluster.local:7777"
diff --git a/.werft/util/werft.ts b/.werft/util/werft.ts
@@ -66,6 +66,9 @@ export class Werft {
         cmd.toString().split("\n").forEach((line: string) => this.log(slice, line))
     }
 
+    /**
+     * Use this when you intend to fail the werft job
+     */
     public fail(slice, err) {
         // Set the status on the span for the slice and also propagate the status to the phase and root span
         // as well so we can query on all phases that had an error regardless of which slice produced the error.
@@ -85,6 +88,18 @@ export class Werft {
         throw err;
     }
 
+    /**
+     * Use this when you intend to fail a single slice, but not the entire Werft job.
+     */
+    public failSlice(slice: string, error: Error) {
+        const span = this.sliceSpans[slice]
+        if (span) {
+            span.end()
+            delete this.sliceSpans[slice]
+        }
+        console.log(`[${slice}|FAIL] ${error}`)
+    }
+
     public done(slice: string) {
         const span = this.sliceSpans[slice]
         if (span) {
