diff --git a/.werft/platform-delete-preview-environments-cron.ts b/.werft/platform-delete-preview-environments-cron.ts index 56ab5edfe7cfea..e4604bf9d470ca 100644 --- a/.werft/platform-delete-preview-environments-cron.ts +++ b/.werft/platform-delete-preview-environments-cron.ts @@ -6,6 +6,7 @@ import { exec } from './util/shell'; import { previewNameFromBranchName } from './util/preview'; import { CORE_DEV_KUBECONFIG_PATH, HARVESTER_KUBECONFIG_PATH } from './jobs/build/const'; import {deleteDNSRecord} from "./util/gcloud"; +import * as VM from "./vm/vm"; // for testing purposes // if set to 'true' it shows only previews that would be deleted @@ -13,11 +14,12 @@ const DRY_RUN = false const SLICES = { CONFIGURE_ACCESS: "Configuring access to relevant resources", + INSTALL_HARVESTER_KUBECONFIG: "Install Harvester kubeconfig", FETCHING_PREVIEW_ENVIRONMENTS: "Fetching preview environments", FETCHING_BRANCHES: "Fetching branches", + CHECKING_FOR_STALE_BRANCHES: "Checking for stale branches", + CHECKING_FOR_NO_DB_ACTIVITY: "Checking for DB activity", DETERMINING_STALE_PREVIEW_ENVIRONMENTS: "Determining stale preview environments", - CHECKING_STALE_BRANCH: (branch: string) => `Checking for commit activity on ${branch}`, - CHECKING_DB_ACTIVITY: (preview: string) => `Checking for DB activity in ${preview}`, DELETING_PREVIEW_ENVIRONMNETS: "Deleting preview environments" } @@ -44,6 +46,173 @@ Tracing.initialize() werft.endAllSpans() }) +class HarvesterPreviewEnvironment { + + // The prefix we use for the namespace + static readonly namespacePrefix: string = "preview-" + + // The name of the namespace that the VM and related resources are in, e.g. preview-my-branch + namespace: string + + // Then name of the preview environment, e.g. my-branch + name: string + + constructor (namespace: string) { + this.namespace = namespace + this.name = namespace.replace(HarvesterPreviewEnvironment.namespacePrefix, "") + } + + async delete(): Promise { + VM.deleteVM({ name: this.name }) + } + + async removeDNSRecords(sliceID: string) { + werft.log(sliceID, "Deleting harvester related DNS records for the preview environment") + await Promise.all([ + deleteDNSRecord('A', `*.ws-dev.${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), + deleteDNSRecord('A', `*.${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), + deleteDNSRecord('A', `${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), + deleteDNSRecord('A', `prometheus-${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), + deleteDNSRecord('TXT', `prometheus-${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), + deleteDNSRecord('A', `grafana-${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), + deleteDNSRecord('TXT', `grafana-${this.name}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID) + ]) + } + + isInactive(): boolean { + // We'll port over the logic from CoreDevPreviewEnvironment later, for now we consider + // Harvester preview environments to never be stale due to inactivity. + const sliceID = SLICES.CHECKING_FOR_NO_DB_ACTIVITY + werft.log(sliceID, `${this.name} (${this.namespace}) - is-inactive=false - Harvester based `) + return false + } + + /** + * Given a branch name it will return the expected namespace of the preview environment + */ + static expectedNamespaceFromBranch(branch: string): string { + const previewName = previewNameFromBranchName(branch) + return `${HarvesterPreviewEnvironment.namespacePrefix}${previewName}` + } +} + +class CoreDevPreviewEnvironment { + + // The prefix we use for the namespace + static readonly namespacePrefix: string = "staging-" + + // The name of the namespace the VM and related resources are in, e.g. preview-my-branch + namespace: string + + name: string + + constructor (namespace: string) { + this.namespace = namespace + this.name = namespace.replace(CoreDevPreviewEnvironment.namespacePrefix, "") + } + + async delete(sliceID: string): Promise { + await wipePreviewEnvironmentAndNamespace(helmInstallName, this.namespace, CORE_DEV_KUBECONFIG_PATH, { slice: sliceID }) + } + + async removeDNSRecords(sliceID: string) { + werft.log(sliceID, "Deleting core-dev related DNS records for the preview environment") + await Promise.all([ + deleteDNSRecord('A', `*.ws-dev.${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('A', `*.${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('A', `${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('A', `prometheus-${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('TXT', `prometheus-${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('A', `grafana-${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('TXT', `grafana-${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('TXT', `_acme-challenge.${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), + deleteDNSRecord('TXT', `_acme-challenge.ws-dev.${this.name}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID) + ]) + } + + /** + * Checks whether or not a preview environment is considered inactive. + * + * It errors on the side of caution, so in case of connection issues etc. it will consider the + * preview environment active. + */ + isInactive(): boolean { + const sliceID = SLICES.CHECKING_FOR_NO_DB_ACTIVITY + try { + const statusNS = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get ns ${this.namespace} -o jsonpath='{.status.phase}'`, { slice: sliceID }) + + if (statusNS != "Active") { + werft.log(sliceID, `${this.name} (${this.namespace}) - is-inactive=false - The namespace is ${statusNS}`) + return false + } + + werft.log(sliceID, `${this.name} (${this.namespace}) - Checking status of the MySQL pod`) + const statusDB = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods mysql-0 -n ${this.namespace} -o jsonpath='{.status.phase}'`, { slice: sliceID}) + const statusDbContainer = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods mysql-0 -n ${this.namespace} -o jsonpath='{.status.containerStatuses.*.ready}'`, { slice: sliceID}) + + if (statusDB.code != 0 || statusDB != "Running" || statusDbContainer == "false") { + werft.log(sliceID, `${this.name} (${this.namespace}) - is-inactive=false - The database is not reachable`) + return false + } + + const dbPassword = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get secret db-password -n ${this.namespace} -o jsonpath='{.data.mysql-root-password}' | base64 -d`, {silent: true}).stdout.trim() + const connectionToDb = `mysql --host=db.${this.namespace}.svc.cluster.local --port=3306 --user=root --database=gitpod -s -N --password=${dbPassword}` + + const latestInstanceTimeout = 48 + const latestInstance = exec(`${connectionToDb} --execute="SELECT creationTime FROM d_b_workspace_instance WHERE creationTime > DATE_SUB(NOW(), INTERVAL '${latestInstanceTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + + const latestUserTimeout = 48 + const latestUser= exec(`${connectionToDb} --execute="SELECT creationDate FROM d_b_user WHERE creationDate > DATE_SUB(NOW(), INTERVAL '${latestUserTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + + const lastModifiedTimeout = 48 + const lastModified= exec(`${connectionToDb} --execute="SELECT _lastModified FROM d_b_user WHERE _lastModified > DATE_SUB(NOW(), INTERVAL '${lastModifiedTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + + const heartbeatTimeout = 48 + const heartbeat= exec(`${connectionToDb} --execute="SELECT lastSeen FROM d_b_workspace_instance_user WHERE lastSeen > DATE_SUB(NOW(), INTERVAL '${heartbeatTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + + const isInactive = (heartbeat.length < 1) && (latestInstance.length < 1) && (latestUser.length < 1) && (lastModified.length < 1) + werft.log(sliceID, `${this.name} (${this.namespace}) - is-inactive=${isInactive}`) + return isInactive + } catch (err) { + werft.log(sliceID, `${this.name} (${this.namespace}) - is-inactive=false - Unable to check DB activity`) + return false + } + } + + + /** + * Given a branch name it will return the expected namespace of the preview environment + */ + static expectedNamespaceFromBranch(branch: string): string { + const previewName = previewNameFromBranchName(branch) + return `${CoreDevPreviewEnvironment.namespacePrefix}${previewName}` + } + +} + +type PreviewEnvironment = CoreDevPreviewEnvironment | HarvesterPreviewEnvironment + +async function getAllPreviewEnvironments(slice: string): Promise { + const coreDevPreviewEnvironments = listAllPreviewNamespaces(CORE_DEV_KUBECONFIG_PATH, {slice: slice}) + .map((namespace: string) => new CoreDevPreviewEnvironment(namespace)) + + const harvesterPreviewEnvironments = exec(`kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} get ns -o=custom-columns=:metadata.name | grep preview-`, { slice, silent: true, async: false }) + .stdout + .trim() + .split("\n") + .map(namespace => new HarvesterPreviewEnvironment(namespace.trim())) + + const all = coreDevPreviewEnvironments.concat(harvesterPreviewEnvironments) + + werft.currentPhaseSpan.setAttributes({ + "preview_environments.counts.core_dev": coreDevPreviewEnvironments.length, + "preview_environments.counts.harvester": harvesterPreviewEnvironments.length, + }) + + // We never want to delete the environment for the main branch. + return all.filter((preview: PreviewEnvironment) => preview.name != "main") +} + async function deletePreviewEnvironments() { werft.phase("Configure access"); @@ -56,11 +225,19 @@ async function deletePreviewEnvironments() { werft.fail(SLICES.CONFIGURE_ACCESS, err) } + werft.phase("Install Harvester kubeconfig"); + try { + exec(`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`, { slice: SLICES.INSTALL_HARVESTER_KUBECONFIG }) + werft.done(SLICES.INSTALL_HARVESTER_KUBECONFIG) + } catch (err) { + werft.fail(SLICES.INSTALL_HARVESTER_KUBECONFIG, err) + } + werft.phase("Fetching preview environments"); - let previews: string[] + let previews: PreviewEnvironment[] try { - previews = listAllPreviewNamespaces(CORE_DEV_KUBECONFIG_PATH, {}); - previews.forEach(previewNs => werft.log(SLICES.FETCHING_PREVIEW_ENVIRONMENTS, previewNs)); + previews = await getAllPreviewEnvironments(SLICES.FETCHING_PREVIEW_ENVIRONMENTS); + previews.forEach((preview: PreviewEnvironment) => werft.log(SLICES.FETCHING_PREVIEW_ENVIRONMENTS, `${preview.name} (${preview.namespace})`)); werft.log(SLICES.FETCHING_PREVIEW_ENVIRONMENTS, `Found ${previews.length} preview environments`) werft.done(SLICES.FETCHING_PREVIEW_ENVIRONMENTS); } catch (err) { @@ -73,40 +250,24 @@ async function deletePreviewEnvironments() { werft.phase("Determining which preview environments are stale"); - const previewNamespaceBasedOnBranches = new Set(branches.map(branch => expectedNamespaceFromBranch(branch))); - - const previewNamespaceBasedOnStaleBranches = new Set(branches - .filter(branch => { - const sliceID = SLICES.CHECKING_STALE_BRANCH(branch) - const lastCommit = exec(`git log origin/${branch} --since=$(date +%Y-%m-%d -d "5 days ago")`, { slice: sliceID }) - const hasRecentCommits = lastCommit.length > 1 - werft.log(sliceID, `Has recent commits: ${hasRecentCommits}`) - werft.done(sliceID) - return !hasRecentCommits - }) - .map(branch => expectedNamespaceFromBranch(branch))) - - const deleteDueToMissingBranch = previews.filter(ns => !previewNamespaceBasedOnBranches.has(ns)) - const deleteDueToNoCommitActivity = previews.filter(ns => previewNamespaceBasedOnStaleBranches.has(ns)) - const deleteDueToNoDBActivity = previews.filter(ns => isInactive(ns)) - const previewsToDelete = new Set([...deleteDueToMissingBranch, ...deleteDueToNoCommitActivity, ...deleteDueToNoDBActivity]) - - if (previewsToDelete.has("staging-main")) { - previewsToDelete.delete("staging-main") - } + const previewsToDelete = await determineStalePreviewEnvironments({ + branches: branches, + previews: previews + }) - if (previewsToDelete.size == 0) { + if (previewsToDelete.length == 0) { werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, "No stale preview environments.") werft.done(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS) return } else { - werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Found ${previewsToDelete.size} stale preview environments`) + werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Found ${previewsToDelete.length} stale preview environments`) + werft.done(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS) } werft.phase("Deleting stale preview environments") if (DRY_RUN) { previewsToDelete.forEach(preview => { - werft.log(SLICES.DELETING_PREVIEW_ENVIRONMNETS, `Would have deleted preview environment ${preview}`) + werft.log(SLICES.DELETING_PREVIEW_ENVIRONMNETS, `Would have deleted preview environment ${preview.name} (${preview.namespace})`) }) werft.done(SLICES.DELETING_PREVIEW_ENVIRONMNETS) return @@ -122,125 +283,99 @@ async function deletePreviewEnvironments() { } } -async function removePreviewEnvironment(previewNamespace: string) { - const sliceID = `Deleting preview ${previewNamespace}` - werft.log(sliceID, `Starting deletion of all resources related to ${previewNamespace}`) - try { - const previewDNSName = previewNamespace.replace('staging-', '') - - // We're running these promises sequentially to make it easier to read the log output. - await removeCertificate(previewNamespace, CORE_DEV_KUBECONFIG_PATH, sliceID) - await removeStagingDNSRecord(previewDNSName, sliceID) - await removePreviewDNSRecord(previewDNSName, sliceID) - await wipePreviewEnvironmentAndNamespace(helmInstallName, previewNamespace, CORE_DEV_KUBECONFIG_PATH, { slice: sliceID }) - werft.done(sliceID) - } catch (e) { - werft.fail(sliceID, e) - } -} - /** - * Checks whether or not a preview environment is considered inactive. + * Determines if preview environemnts are stale and should be deleted * - * It errors on the side of caution, so in case of connection issues etc. it will consider the - * preview environment active. + * As we don't have a mapping from preview environnment -> Git branch we have to + * go about this in a bit of a backwards way. + * + * Based on the active git branches we compute a set of "expected" preview environments + * and then use that to compare with the "live" preview environemnts to decide which + * ones to keep */ -function isInactive(previewNS: string): boolean { - const sliceID = SLICES.CHECKING_DB_ACTIVITY(previewNS) - try { - werft.log(sliceID, "Checking namespace status") - const statusNS = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get ns ${previewNS} -o jsonpath='{.status.phase}'`, { slice: sliceID }) - - if (statusNS != "Active") { - werft.log(sliceID, `Is inactive: false - The namespace is ${statusNS}`) - werft.done(sliceID) - return false +async function determineStalePreviewEnvironments(options: {previews: PreviewEnvironment[], branches: string[]}): Promise { + + const {branches, previews} = options + + // The set of namespaces that we would expect based on the open branches. + // This contains both the core-dev and the harvester namespaces as we only use this set for + // testing membership in situations where we don't care if the preview environment is based on + // core-dev or harvester. + const previewNamespaceBasedOnBranches = new Set(branches.flatMap(branch => [ + CoreDevPreviewEnvironment.expectedNamespaceFromBranch(branch), + HarvesterPreviewEnvironment.expectedNamespaceFromBranch(branch) + ])); + + // The set of namespaces where the underlying branch is considered stale + // This contains both core-dev and harvester namespaces, see above. + werft.log(SLICES.CHECKING_FOR_STALE_BRANCHES, `Checking commit activity on ${branches.length} branches`) + const previewNamespaceBasedOnStaleBranches = new Set(branches + .filter(branch => { + const lastCommit = exec(`git log origin/${branch} --since=$(date +%Y-%m-%d -d "5 days ago")`, { silent: true }) + const hasRecentCommits = lastCommit.length > 1 + werft.log(SLICES.CHECKING_FOR_STALE_BRANCHES, `${branch} has-recent-commits=${hasRecentCommits}`) + return !hasRecentCommits + }) + .flatMap((branch: string) => [ + CoreDevPreviewEnvironment.expectedNamespaceFromBranch(branch), + HarvesterPreviewEnvironment.expectedNamespaceFromBranch(branch) + ])) + werft.done(SLICES.CHECKING_FOR_STALE_BRANCHES) + + werft.log(SLICES.CHECKING_FOR_NO_DB_ACTIVITY, `Checking ${previews.length} preview environments for DB activity`) + const previewNamespacesWithNoDBActivity = new Set( + previews + .filter((preview) => preview.isInactive()) + .map((preview) => preview.namespace) + ) + + werft.done(SLICES.CHECKING_FOR_NO_DB_ACTIVITY) + + const previewsToDelete = previews.filter((preview: PreviewEnvironment) => { + if (!previewNamespaceBasedOnBranches.has(preview.namespace)) { + werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to missing branch`) + return true } - werft.log(sliceID, "Checking status of the MySQL pod") - const statusDB = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods mysql-0 -n ${previewNS} -o jsonpath='{.status.phase}'`, { slice: sliceID}) - const statusDbContainer = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods mysql-0 -n ${previewNS} -o jsonpath='{.status.containerStatuses.*.ready}'`, { slice: sliceID}) - - if (statusDB.code != 0 || statusDB != "Running" || statusDbContainer == "false") { - werft.log(sliceID, "Is inactive: false - The database is not reachable") - werft.done(sliceID) - return false + if (previewNamespaceBasedOnStaleBranches.has(preview.namespace)) { + werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to no recent commit activity`) + return true } - const dbPassword = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get secret db-password -n ${previewNS} -o jsonpath='{.data.mysql-root-password}' | base64 -d`, {silent: true}).stdout.trim() - const connectionToDb = `mysql --host=db.${previewNS}.svc.cluster.local --port=3306 --user=root --database=gitpod -s -N --password=${dbPassword}` - - const latestInstanceTimeout = 48 - const latestInstance = exec(`${connectionToDb} --execute="SELECT creationTime FROM d_b_workspace_instance WHERE creationTime > DATE_SUB(NOW(), INTERVAL '${latestInstanceTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) - - const latestUserTimeout = 48 - const latestUser= exec(`${connectionToDb} --execute="SELECT creationDate FROM d_b_user WHERE creationDate > DATE_SUB(NOW(), INTERVAL '${latestUserTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + if (previewNamespacesWithNoDBActivity.has(preview.namespace)) { + werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) stale due to no recent DB activity`) + return true + } - const lastModifiedTimeout = 48 - const lastModified= exec(`${connectionToDb} --execute="SELECT _lastModified FROM d_b_user WHERE _lastModified > DATE_SUB(NOW(), INTERVAL '${lastModifiedTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + werft.log(SLICES.DETERMINING_STALE_PREVIEW_ENVIRONMENTS, `Considering ${preview.name} (${preview.namespace}) active`) + return false + }) - const heartbeatTimeout = 48 - const heartbeat= exec(`${connectionToDb} --execute="SELECT lastSeen FROM d_b_workspace_instance_user WHERE lastSeen > DATE_SUB(NOW(), INTERVAL '${heartbeatTimeout}' HOUR) LIMIT 1"`, { slice: sliceID}) + return previewsToDelete +} - const isInactive = (heartbeat.length < 1) && (latestInstance.length < 1) && (latestUser.length < 1) && (lastModified.length < 1) - werft.log(sliceID, `Is inactive: ${isInactive}`) - werft.done(sliceID) - return isInactive - } catch (err) { - werft.log(sliceID, "Is inactive: false - Unable to check DB activity") +async function removePreviewEnvironment(previewEnvironment: PreviewEnvironment) { + const sliceID = `Deleting preview ${previewEnvironment.name}` + werft.log(sliceID, `Starting deletion of all resources related to ${previewEnvironment.name}`) + try { + // We're running these promises sequentially to make it easier to read the log output. + await removeCertificate(previewEnvironment.name, CORE_DEV_KUBECONFIG_PATH, sliceID) + await previewEnvironment.removeDNSRecords(sliceID) + await previewEnvironment.delete(sliceID) werft.done(sliceID) - return false + } catch (e) { + werft.failSlice(sliceID, e) } } async function removeCertificate(preview: string, kubectlConfig: string, slice: string) { - exec(`kubectl --kubeconfig ${kubectlConfig} -n certs delete cert ${preview}`, {slice: slice}) -} - -// remove DNS records for core-dev-based preview environments -async function removeStagingDNSRecord(preview: string, sliceID: string) { - werft.log(sliceID, "Deleting core-dev related DNS records for the preview environment") - await Promise.all([ - deleteDNSRecord('A', `*.ws-dev.${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('A', `*.${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('A', `${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('A', `prometheus-${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('TXT', `prometheus-${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('A', `grafana-${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('TXT', `grafana-${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('TXT', `_acme-challenge.${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID), - deleteDNSRecord('TXT', `_acme-challenge.ws-dev.${preview}.staging.gitpod-dev.com`, 'gitpod-dev', 'gitpod-dev-com', sliceID) - ]) -} - -// remove DNS records for harvester-based preview environments -async function removePreviewDNSRecord(preview: string, sliceID: string) { - werft.log(sliceID, "Deleting harvester related DNS records for the preview environment") - await Promise.all([ - deleteDNSRecord('A', `*.ws-dev.${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), - deleteDNSRecord('A', `*.${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), - deleteDNSRecord('A', `${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), - deleteDNSRecord('A', `prometheus-${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), - deleteDNSRecord('TXT', `prometheus-${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), - deleteDNSRecord('A', `grafana-${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID), - deleteDNSRecord('TXT', `grafana-${preview}.preview.gitpod-dev.com`, 'gitpod-core-dev', 'preview-gitpod-dev-com', sliceID) - ]) + return exec(`kubectl --kubeconfig ${kubectlConfig} -n certs delete --ignore-not-found=true cert ${preview}`, {slice: slice, async: true}) } async function cleanLoadbalancer() { - const prepPhase = "prep clean loadbalancers" const fetchPhase = "fetching unuse loadbalancer" const deletionPhase = "deleting unused load balancers" - werft.phase(prepPhase); - try { - exec(`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`) - } catch (err) { - werft.fail(prepPhase, err) - } - werft.done(prepPhase) - - werft.phase(fetchPhase); let lbsToDelete: string[] try { @@ -271,8 +406,3 @@ async function cleanLoadbalancer() { function getAllBranches(): string[] { return exec(`git branch -r | grep -v '\\->' | sed "s,\\x1B\\[[0-9;]*[a-zA-Z],,g" | while read remote; do echo "\${remote#origin/}"; done`).stdout.trim().split('\n'); } - -function expectedNamespaceFromBranch(branch: string): string { - const previewName = previewNameFromBranchName(branch) - return `staging-${previewName}` -} diff --git a/.werft/platform-delete-preview-environments-cron.yaml b/.werft/platform-delete-preview-environments-cron.yaml index 1af1571f8409e6..e76769472d7c9b 100644 --- a/.werft/platform-delete-preview-environments-cron.yaml +++ b/.werft/platform-delete-preview-environments-cron.yaml @@ -19,6 +19,9 @@ pod: - name: harvester-kubeconfig secret: secretName: harvester-kubeconfig + - name: harvester-k3s-dockerhub-pull-account + secret: + secretName: harvester-k3s-dockerhub-pull-account containers: - name: build image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go.4 @@ -33,6 +36,8 @@ pod: readOnly: true - name: harvester-kubeconfig mountPath: /mnt/secrets/harvester-kubeconfig + - name: harvester-k3s-dockerhub-pull-account + mountPath: /mnt/secrets/harvester-k3s-dockerhub-pull-account env: - name: WERFT_HOST value: "werft.werft.svc.cluster.local:7777" diff --git a/.werft/util/werft.ts b/.werft/util/werft.ts index 954ef2aae285a0..d01017561ef569 100644 --- a/.werft/util/werft.ts +++ b/.werft/util/werft.ts @@ -66,6 +66,9 @@ export class Werft { cmd.toString().split("\n").forEach((line: string) => this.log(slice, line)) } + /** + * Use this when you intend to fail the werft job + */ public fail(slice, err) { // Set the status on the span for the slice and also propagate the status to the phase and root span // as well so we can query on all phases that had an error regardless of which slice produced the error. @@ -85,6 +88,22 @@ export class Werft { throw err; } + /** + * Use this when you intend to fail a single slice, but not the entire Werft job. + */ + public failSlice(slice: string, error: Error) { + const span = this.sliceSpans[slice] + if (span) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error.message + }) + span.end() + delete this.sliceSpans[slice] + } + console.log(`[${slice}|FAIL] ${error}`) + } + public done(slice: string) { const span = this.sliceSpans[slice] if (span) {