[server] Purge Workspaces at some point

Author: geropl

components/gitpod-db/src/typeorm/workspace-db-impl.ts

@@ -52,6 +52,7 @@ import { DBPrebuiltWorkspace } from "./entity/db-prebuilt-workspace";
 import { DBPrebuiltWorkspaceUpdatable } from "./entity/db-prebuilt-workspace-updatable";
 import { BUILTIN_WORKSPACE_PROBE_USER_ID } from "../user-db";
 import { DBPrebuildInfo } from "./entity/db-prebuild-info-entry";
+import { daysEarlier } from "@gitpod/gitpod-protocol/lib/util/timeutil";
 
 type RawTo<T> = (instance: WorkspaceInstance, ws: Workspace) => T;
 interface OrderBy {
@@ -592,6 +593,22 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
         return dbResults as WorkspaceAndOwner[];
     }
 
+    public async findWorkspacesForPurging(
+        minContentDeletionTimeInDays: number,
+        limit: number,
+        now: Date,
+    ): Promise<WorkspaceAndOwner[]> {
+        const minPurgeTime = daysEarlier(now, minContentDeletionTimeInDays).toISOString();
+        const repo = await this.getWorkspaceRepo();
+        const qb = repo
+            .createQueryBuilder("ws")
+            .select(["ws.id", "ws.ownerId"])
+            .where(`ws.contentDeletedTime != ''`)
+            .andWhere(`ws.contentDeletedTime < :minPurgeTime`, { minPurgeTime })
+            .limit(limit);
+        return await qb.getMany();
+    }
+
     public async findWorkspacesForContentDeletion(
         minSoftDeletedTimeInDays: number,
         limit: number,
@@ -964,8 +981,8 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
      *       around to deleting them.
      */
     public async hardDeleteWorkspace(workspaceId: string): Promise<void> {
-        await (await this.getWorkspaceRepo()).update(workspaceId, { deleted: true });
         await (await this.getWorkspaceInstanceRepo()).update({ workspaceId }, { deleted: true });
+        await (await this.getWorkspaceRepo()).update(workspaceId, { deleted: true });
     }
 
     public async findAllWorkspaces(

components/gitpod-db/src/workspace-db.spec.db.ts

@@ -662,6 +662,7 @@ class WorkspaceDBSpec {
             ]);
         }
     }
+
     @test(timeout(10000))
     public async testFindVolumeSnapshotWorkspacesForGC() {
         await this.threeVolumeSnapshotsForTwoWorkspaces();
@@ -726,5 +727,64 @@ class WorkspaceDBSpec {
             workspaceId: workspaceId2,
         });
     }
+
+    @test(timeout(10000))
+    public async findWorkspacesForPurging() {
+        const creationTime = "2018-01-01T00:00:00.000Z";
+        const ownerId = "1221423";
+        const purgeDate = new Date("2019-02-01T00:00:00.000Z");
+        const d20180202 = "2018-02-02T00:00:00.000Z";
+        const d20180201 = "2018-02-01T00:00:00.000Z";
+        const d20180131 = "2018-01-31T00:00:00.000Z";
+        await Promise.all([
+            this.db.store({
+                id: "1",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/inactive",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: d20180131,
+            }),
+            this.db.store({
+                id: "2",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/active",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: d20180201,
+            }),
+            this.db.store({
+                id: "3",
+                creationTime,
+                description: "something",
+                contextURL: "http://github.com/myorg/active",
+                ownerId,
+                context: {
+                    title: "my title",
+                },
+                config: {},
+                type: "regular",
+                contentDeletedTime: d20180202,
+            }),
+        ]);
+
+        const wsIds = await this.db.findWorkspacesForPurging(365, 1000, purgeDate);
+        expect(wsIds).to.deep.equal([
+            {
+                id: "1",
+                ownerId,
+            },
+        ]);
+    }
 }
 module.exports = new WorkspaceDBSpec();

components/gitpod-db/src/workspace-db.ts

@@ -108,6 +108,11 @@ export interface WorkspaceDB {
         minSoftDeletedTimeInDays: number,
         limit: number,
     ): Promise<WorkspaceOwnerAndSoftDeleted[]>;
+    findWorkspacesForPurging(
+        minContentDeletionTimeInDays: number,
+        limit: number,
+        now: Date,
+    ): Promise<WorkspaceAndOwner[]>;
     findPrebuiltWorkspacesForGC(daysUnused: number, limit: number): Promise<WorkspaceAndOwner[]>;
     findAllWorkspaces(
         offset: number,

components/gitpod-protocol/src/util/timeutil.spec.ts

@@ -7,7 +7,7 @@
 import * as chai from "chai";
 const expect = chai.expect;
 import { suite, test } from "mocha-typescript";
-import { oneMonthLater } from "./timeutil";
+import { daysEarlier, oneMonthLater } from "./timeutil";
 
 @suite()
 export class TimeutilSpec {
@@ -39,4 +39,25 @@ export class TimeutilSpec {
         const later = oneMonthLater(from.toISOString(), day);
         expect(later, `expected ${later} to be equal ${expectation}`).to.be.equal(expectation.toISOString());
     }
+
+    @test
+    testDaysEarlier() {
+        const tests: { date: Date; daysEarlier: number; expectation: string }[] = [
+            {
+                date: new Date("2021-07-13T00:00:00.000Z"),
+                daysEarlier: 365,
+                expectation: "2020-07-13T00:00:00.000Z",
+            },
+            {
+                date: new Date("2019-02-01T00:00:00.000Z"),
+                daysEarlier: 365,
+                expectation: "2018-02-01T00:00:00.000Z",
+            },
+        ];
+
+        for (const t of tests) {
+            const actual = daysEarlier(t.date, t.daysEarlier);
+            expect(actual.toISOString()).to.equal(t.expectation, `expected ${actual} to be equal ${t.expectation}`);
+        }
+    }
 }

components/gitpod-protocol/src/util/timeutil.ts

@@ -47,6 +47,12 @@ export const orderAsc = (d1: string, d2: string): number => liftDate(d1, d2, (d1
 export const liftDate1 = <T>(d1: string, f: (d1: Date) => T): T => f(new Date(d1));
 export const liftDate = <T>(d1: string, d2: string, f: (d1: Date, d2: Date) => T): T => f(new Date(d1), new Date(d2));
 
+export const daysEarlier = (fromDate: Date, days: number): Date => {
+    const t1 = new Date(fromDate.getTime());
+    t1.setUTCDate(t1.getDate() - days);
+    return t1;
+};
+
 export function hoursBefore(date: string, hours: number): string {
     const result = new Date(date);
     result.setHours(result.getHours() - hours);

components/server/src/config.ts

@@ -48,11 +48,27 @@ export interface WorkspaceDefaults {
 export interface WorkspaceGarbageCollection {
     disabled: boolean;
     startDate: number;
+
+    /** The maximum amount of workspaces that are marked as 'softDeleted' in one go */
     chunkLimit: number;
+
+    /** The minimal age of a workspace before it is marked as 'softDeleted' (= hidden for the user) */
     minAgeDays: number;
+
+    /** The minimal age of a prebuild (incl. workspace) before it's content is deleted (+ marked as 'softDeleted') */
     minAgePrebuildDays: number;
+
+    /** The minimal number of days a workspace has to stay in 'softDeleted' before it's content is deleted */
     contentRetentionPeriodDays: number;
+
+    /** The maximum amount of workspaces whose content is deleted in one go */
     contentChunkLimit: number;
+
+    /** The minimal number of days a workspace has to stay in 'contentDeleted' before it's purged from the DB */
+    purgeRetentionPeriodDays: number;
+
+    /** The maximum amount of workspaces which are purged in one go */
+    purgeChunkLimit: number;
 }
 
 /**

components/server/src/workspace/garbage-collector.ts

@@ -44,6 +44,9 @@ export class WorkspaceGarbageCollector {
             this.deleteWorkspaceContentAfterRetentionPeriod().catch((err) =>
                 log.error("wsgc: error during content deletion", err),
             );
+            this.purgeWorkspacesAfterPurgeRetentionPeriod().catch((err) =>
+                log.error("wsgc: error during hard deletion of workspaces", err),
+            );
             this.deleteOldPrebuilds().catch((err) => log.error("wsgc: error during prebuild deletion", err));
             this.deleteOutdatedVolumeSnapshots().catch((err) =>
                 log.error("wsgc: error during volume snapshot gc deletion", err),
@@ -105,6 +108,34 @@ export class WorkspaceGarbageCollector {
         }
     }
 
+    /**
+     * This method is meant to purge all traces of a Workspace and it's WorkspaceInstances from the DB
+     */
+    protected async purgeWorkspacesAfterPurgeRetentionPeriod() {
+        const span = opentracing.globalTracer().startSpan("purgeWorkspacesAfterPurgeRetentionPeriod");
+        try {
+            const now = new Date();
+            const workspaces = await this.workspaceDB
+                .trace({ span })
+                .findWorkspacesForPurging(
+                    this.config.workspaceGarbageCollection.purgeRetentionPeriodDays,
+                    this.config.workspaceGarbageCollection.purgeChunkLimit,
+                    now,
+                );
+            const deletes = await Promise.all(
+                workspaces.map((ws) => this.deletionService.hardDeleteWorkspace({ span }, ws.id)),
+            );
+
+            log.info(`wsgc: successfully purged ${deletes.length} workspaces`);
+            span.addTags({ nrOfCollectedWorkspaces: deletes.length });
+        } catch (err) {
+            TraceContext.setError({ span }, err);
+            throw err;
+        } finally {
+            span.finish();
+        }
+    }
+
     protected async deleteOldPrebuilds() {
         const span = opentracing.globalTracer().startSpan("deleteOldPrebuilds");
         try {
@@ -128,7 +159,7 @@ export class WorkspaceGarbageCollector {
         }
     }
 
-    // finds volume snapshots that have been superceded by newer volume snapshot and removes them
+    // finds volume snapshots that have been superseded by newer volume snapshot and removes them
     protected async deleteOutdatedVolumeSnapshots() {
         const span = opentracing.globalTracer().startSpan("deleteOutdatedVolumeSnapshots");
         try {

components/server/src/workspace/gitpod-server-impl.ts

@@ -1133,7 +1133,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
             try {
                 await this.guardAccess({ kind: "workspace", subject: workspace }, "create");
             } catch (err) {
-                await this.workspaceDb.trace(ctx).hardDeleteWorkspace(workspace.id);
+                await this.workspaceDeletionService.hardDeleteWorkspace(ctx, workspace.id);
                 throw err;
             }
 

components/server/src/workspace/workspace-deletion-service.ts

@@ -46,6 +46,17 @@ export class WorkspaceDeletionService {
         });
     }
 
+    /**
+     * This *hard deletes* the workspace entry and all corresponding workspace-instances, by triggering a db-sync mechanism that purges it from the DB.
+     * Note: when this function returns that doesn't mean that the entries are actually gone yet, that might still take a short while until db-sync comes
+     *       around to deleting them.
+     * @param ctx
+     * @param workspaceId
+     */
+    public async hardDeleteWorkspace(ctx: TraceContext, workspaceId: string): Promise<void> {
+        await this.db.trace(ctx).hardDeleteWorkspace(workspaceId);
+    }
+
     /**
      * This method garbageCollects a workspace. It deletes its contents and sets the workspaces 'contentDeletedTime'
      * @param ctx

install/installer/pkg/components/server/configmap.go

@@ -178,12 +178,14 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 		DefinitelyGpDisabled: ctx.Config.DisableDefinitelyGP,
 		GitHubApp:            githubApp,
 		WorkspaceGarbageCollection: WorkspaceGarbageCollection{
-			ChunkLimit:                 1000,
-			ContentChunkLimit:          1000,
-			ContentRetentionPeriodDays: 21,
 			Disabled:                   disableWsGarbageCollection,
 			MinAgeDays:                 14,
 			MinAgePrebuildDays:         7,
+			ChunkLimit:                 1000,
+			ContentRetentionPeriodDays: 21,
+			ContentChunkLimit:          1000,
+			PurgeRetentionPeriodDays:   365,
+			PurgeChunkLimit:            1000,
 		},
 		EnableLocalApp: enableLocalApp,
 		AuthProviderConfigFiles: func() []string {

install/installer/pkg/components/server/types.go

@@ -102,6 +102,8 @@ type WorkspaceGarbageCollection struct {
 	MinAgePrebuildDays         int32 `json:"minAgePrebuildDays"`
 	ContentRetentionPeriodDays int32 `json:"contentRetentionPeriodDays"`
 	ContentChunkLimit          int32 `json:"contentChunkLimit"`
+	PurgeRetentionPeriodDays   int32 `json:"purgeRetentionPeriodDays"`
+	PurgeChunkLimit            int32 `json:"purgeChunkLimit"`
 }
 
 type GitHubApp struct {