[server] Delete Workspaces & Workspace Instances after configured period

Author: geropl

components/gitpod-db/src/typeorm/workspace-db-impl.ts

@@ -52,6 +52,7 @@ import { DBPrebuiltWorkspace } from "./entity/db-prebuilt-workspace";
 import { DBPrebuiltWorkspaceUpdatable } from "./entity/db-prebuilt-workspace-updatable";
 import { BUILTIN_WORKSPACE_PROBE_USER_ID } from "../user-db";
 import { DBPrebuildInfo } from "./entity/db-prebuild-info-entry";
+import { daysBefore } from "@gitpod/gitpod-protocol/lib/util/timeutil";
 
 type RawTo<T> = (instance: WorkspaceInstance, ws: Workspace) => T;
 interface OrderBy {
@@ -592,6 +593,22 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
         return dbResults as WorkspaceAndOwner[];
     }
 
+    public async findWorkspacesForPurging(
+        minContentDeletionTimeInDays: number,
+        limit: number,
+        now: Date,
+    ): Promise<WorkspaceAndOwner[]> {
+        const minPurgeTime = daysBefore(now.toISOString(), minContentDeletionTimeInDays);
+        const repo = await this.getWorkspaceRepo();
+        const qb = repo
+            .createQueryBuilder("ws")
+            .select(["ws.id", "ws.ownerId"])
+            .where(`ws.contentDeletedTime != ''`)
+            .andWhere(`ws.contentDeletedTime < :minPurgeTime`, { minPurgeTime })
+            .limit(limit);
+        return await qb.getMany();
+    }
+
     public async findWorkspacesForContentDeletion(
         minSoftDeletedTimeInDays: number,
         limit: number,
@@ -964,8 +981,8 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
      *       around to deleting them.
      */
     public async hardDeleteWorkspace(workspaceId: string): Promise<void> {
-        await (await this.getWorkspaceRepo()).update(workspaceId, { deleted: true });
         await (await this.getWorkspaceInstanceRepo()).update({ workspaceId }, { deleted: true });
+        await (await this.getWorkspaceRepo()).update(workspaceId, { deleted: true });
     }
 
     public async findAllWorkspaces(

components/gitpod-db/src/workspace-db.spec.db.ts

@@ -662,6 +662,7 @@ class WorkspaceDBSpec {
             ]);
         }
     }
+
     @test(timeout(10000))
     public async testFindVolumeSnapshotWorkspacesForGC() {
         await this.threeVolumeSnapshotsForTwoWorkspaces();
@@ -726,5 +727,77 @@ class WorkspaceDBSpec {
             workspaceId: workspaceId2,
         });
     }
+
+    @test(timeout(10000))
+    public async findWorkspacesForPurging() {
+        const creationTime = "2018-01-01T00:00:00.000Z";
+        const ownerId = "1221423";
+        const purgeDate = new Date("2019-02-01T00:00:00.000Z");
+        const d20180202 = "2018-02-02T00:00:00.000Z";
+        const d20180201 = "2018-02-01T00:00:00.000Z";
+        const d20180131 = "2018-01-31T00:00:00.000Z";
+        await Promise.all([
+            this.db.store({
+                id: "1",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/inactive",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: d20180131,
+            }),
+            this.db.store({
+                id: "2",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/active",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: d20180201,
+            }),
+            this.db.store({
+                id: "3",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/active",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: d20180202,
+            }),
+            this.db.store({
+                id: "4",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/active",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: undefined,
+            }),
+        ]);
+
+        const wsIds = await this.db.findWorkspacesForPurging(365, 1000, purgeDate);
+        expect(wsIds).to.deep.equal([
+            {
+                id: "1",
+                ownerId,
+            },
+        ]);
+    }
 }
 module.exports = new WorkspaceDBSpec();

components/gitpod-db/src/workspace-db.ts

@@ -108,6 +108,11 @@ export interface WorkspaceDB {
         minSoftDeletedTimeInDays: number,
         limit: number,
     ): Promise<WorkspaceOwnerAndSoftDeleted[]>;
+    findWorkspacesForPurging(
+        minContentDeletionTimeInDays: number,
+        limit: number,
+        now: Date,
+    ): Promise<WorkspaceAndOwner[]>;
     findPrebuiltWorkspacesForGC(daysUnused: number, limit: number): Promise<WorkspaceAndOwner[]>;
     findAllWorkspaces(
         offset: number,

components/gitpod-protocol/src/util/timeutil.spec.ts

@@ -45,4 +45,25 @@ export class TimeutilSpec {
         const later = oneMonthLater(from.toISOString(), day);
         expect(later, `expected ${later} to be equal ${expectation}`).to.be.equal(expectation.toISOString());
     }
+
+    @test
+    testDaysBefore2() {
+        const tests: { date: Date; daysEarlier: number; expectation: string }[] = [
+            {
+                date: new Date("2021-07-13T00:00:00.000Z"),
+                daysEarlier: 365,
+                expectation: "2020-07-13T00:00:00.000Z",
+            },
+            {
+                date: new Date("2019-02-01T00:00:00.000Z"),
+                daysEarlier: 365,
+                expectation: "2018-02-01T00:00:00.000Z",
+            },
+        ];
+
+        for (const t of tests) {
+            const actual = daysBefore(t.date.toISOString(), t.daysEarlier);
+            expect(actual).to.equal(t.expectation, `expected ${actual} to be equal ${t.expectation}`);
+        }
+    }
 }

components/server/src/config.ts

@@ -43,11 +43,30 @@ export interface WorkspaceDefaults {
 export interface WorkspaceGarbageCollection {
     disabled: boolean;
     startDate: number;
+
+    /** The number of seconds between a run and the next */
+    intervalSeconds: number;
+
+    /** The maximum amount of workspaces that are marked as 'softDeleted' in one go */
     chunkLimit: number;
+
+    /** The minimal age of a workspace before it is marked as 'softDeleted' (= hidden for the user) */
     minAgeDays: number;
+
+    /** The minimal age of a prebuild (incl. workspace) before it's content is deleted (+ marked as 'softDeleted') */
     minAgePrebuildDays: number;
+
+    /** The minimal number of days a workspace has to stay in 'softDeleted' before it's content is deleted */
     contentRetentionPeriodDays: number;
+
+    /** The maximum amount of workspaces whose content is deleted in one go */
     contentChunkLimit: number;
+
+    /** The minimal number of days a workspace has to stay in 'contentDeleted' before it's purged from the DB */
+    purgeRetentionPeriodDays: number;
+
+    /** The maximum amount of workspaces which are purged in one go */
+    purgeChunkLimit: number;
 }
 
 /**

components/server/src/prometheus-metrics.ts

@@ -166,3 +166,13 @@ const prebuildsStartedTotal = new prometheusClient.Counter({
 export function increasePrebuildsStartedCounter() {
     prebuildsStartedTotal.inc();
 }
+
+const workspacesPurgedTotal = new prometheusClient.Counter({
+    name: "gitpod_server_workspaces_purged_total",
+    help: "Counter of workspaces hard deleted by periodic job running on server.",
+    registers: [prometheusClient.register],
+});
+
+export function reportWorkspacePurged() {
+    workspacesPurgedTotal.inc();
+}

components/server/src/workspace/garbage-collector.ts

@@ -34,7 +34,10 @@ export class WorkspaceGarbageCollector {
                 dispose: () => {},
             };
         }
-        return repeat(async () => this.garbageCollectWorkspacesIfLeader(), 30 * 60 * 1000);
+        return repeat(
+            async () => this.garbageCollectWorkspacesIfLeader(),
+            this.config.workspaceGarbageCollection.intervalSeconds * 1000,
+        );
     }
 
     public async garbageCollectWorkspacesIfLeader() {
@@ -44,6 +47,9 @@ export class WorkspaceGarbageCollector {
             this.deleteWorkspaceContentAfterRetentionPeriod().catch((err) =>
                 log.error("wsgc: error during content deletion", err),
             );
+            this.purgeWorkspacesAfterPurgeRetentionPeriod().catch((err) =>
+                log.error("wsgc: error during hard deletion of workspaces", err),
+            );
             this.deleteOldPrebuilds().catch((err) => log.error("wsgc: error during prebuild deletion", err));
             this.deleteOutdatedVolumeSnapshots().catch((err) =>
                 log.error("wsgc: error during volume snapshot gc deletion", err),
@@ -105,6 +111,34 @@ export class WorkspaceGarbageCollector {
         }
     }
 
+    /**
+     * This method is meant to purge all traces of a Workspace and it's WorkspaceInstances from the DB
+     */
+    protected async purgeWorkspacesAfterPurgeRetentionPeriod() {
+        const span = opentracing.globalTracer().startSpan("purgeWorkspacesAfterPurgeRetentionPeriod");
+        try {
+            const now = new Date();
+            const workspaces = await this.workspaceDB
+                .trace({ span })
+                .findWorkspacesForPurging(
+                    this.config.workspaceGarbageCollection.purgeRetentionPeriodDays,
+                    this.config.workspaceGarbageCollection.purgeChunkLimit,
+                    now,
+                );
+            const deletes = await Promise.all(
+                workspaces.map((ws) => this.deletionService.hardDeleteWorkspace({ span }, ws.id)),
+            );
+
+            log.info(`wsgc: successfully purged ${deletes.length} workspaces`);
+            span.addTags({ nrOfCollectedWorkspaces: deletes.length });
+        } catch (err) {
+            TraceContext.setError({ span }, err);
+            throw err;
+        } finally {
+            span.finish();
+        }
+    }
+
     protected async deleteOldPrebuilds() {
         const span = opentracing.globalTracer().startSpan("deleteOldPrebuilds");
         try {
@@ -128,7 +162,7 @@ export class WorkspaceGarbageCollector {
         }
     }
 
-    // finds volume snapshots that have been superceded by newer volume snapshot and removes them
+    // finds volume snapshots that have been superseded by newer volume snapshot and removes them
     protected async deleteOutdatedVolumeSnapshots() {
         const span = opentracing.globalTracer().startSpan("deleteOutdatedVolumeSnapshots");
         try {

components/server/src/workspace/gitpod-server-impl.ts

@@ -1166,7 +1166,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
             try {
                 await this.guardAccess({ kind: "workspace", subject: workspace }, "create");
             } catch (err) {
-                await this.workspaceDb.trace(ctx).hardDeleteWorkspace(workspace.id);
+                await this.workspaceDeletionService.hardDeleteWorkspace(ctx, workspace.id);
                 throw err;
             }
 

components/server/src/workspace/workspace-deletion-service.ts

@@ -19,6 +19,7 @@ import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
 import { DeleteVolumeSnapshotRequest } from "@gitpod/ws-manager/lib";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { reportWorkspacePurged } from "../prometheus-metrics";
 
 @injectable()
 export class WorkspaceDeletionService {
@@ -46,6 +47,19 @@ export class WorkspaceDeletionService {
         });
     }
 
+    /**
+     * This *hard deletes* the workspace entry and all corresponding workspace-instances, by triggering a db-sync mechanism that purges it from the DB.
+     * Note: when this function returns that doesn't mean that the entries are actually gone yet, that might still take a short while until db-sync comes
+     *       around to deleting them.
+     * @param ctx
+     * @param workspaceId
+     */
+    public async hardDeleteWorkspace(ctx: TraceContext, workspaceId: string): Promise<void> {
+        await this.db.trace(ctx).hardDeleteWorkspace(workspaceId);
+        log.info(`Purged Workspace ${workspaceId} and all WorkspaceInstances for this workspace`, { workspaceId });
+        reportWorkspacePurged();
+    }
+
     /**
      * This method garbageCollects a workspace. It deletes its contents and sets the workspaces 'contentDeletedTime'
      * @param ctx