From ab23021893cc4d284e9ef4ceed9995124f0a212f Mon Sep 17 00:00:00 2001 From: Sven Efftinge Date: Thu, 27 Oct 2022 02:38:49 +0000 Subject: [PATCH] [content-service] incremental workspace init if we initialize the workspace from an outdated prebuild, we need the regular workspace tasks (including init) should run. --- .../pkg/initializer/prebuild.go | 27 ++- .../components/ws-manager/prebuild_test.go | 156 ++++++++++++++++++ 2 files changed, 178 insertions(+), 5 deletions(-) diff --git a/components/content-service/pkg/initializer/prebuild.go b/components/content-service/pkg/initializer/prebuild.go index 51262af1f16cda..e80c286d8b12d8 100644 --- a/components/content-service/pkg/initializer/prebuild.go +++ b/components/content-service/pkg/initializer/prebuild.go @@ -81,14 +81,20 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp // at this point we're actually a prebuild initialiser because we've been able to restore // the prebuild. + src = csapi.WorkspaceInitFromPrebuild // make sure we're on the correct branch for _, gi := range p.Git { - err = runGitInit(ctx, gi) + + commitChanged, err := runGitInit(ctx, gi) if err != nil { return src, err } + if commitChanged { + // head commit has changed, so it's an outdated prebuild, which we treat as other + src = csapi.WorkspaceInitFromOther + } } log.Debug("Initialized workspace with prebuilt snapshot") return @@ -108,7 +114,7 @@ func clearWorkspace(location string) error { return nil } -func runGitInit(ctx context.Context, gInit *GitInitializer) (err error) { +func runGitInit(ctx context.Context, gInit *GitInitializer) (commitChanged bool, err error) { span, ctx := opentracing.StartSpanFromContext(ctx, "runGitInit") span.LogFields( tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(gInit.Location))), @@ -125,14 +131,25 @@ func runGitInit(ctx context.Context, gInit *GitInitializer) (err error) { } else { // git returned a non-zero exit code because of some reason we did not anticipate or an actual failure. log.WithError(err).WithField("output", string(out)).Error("unexpected git stash error") - return xerrors.Errorf("prebuild initializer: %w", err) + return commitChanged, xerrors.Errorf("prebuild initializer: %w", err) } } didStash := !strings.Contains(string(out), "No local changes to save") + statusBefore, err := gInit.Status(ctx) + if err != nil { + log.WithError(err).Warn("couldn't run git status - continuing") + } err = checkGitStatus(gInit.realizeCloneTarget(ctx)) if err != nil { - return xerrors.Errorf("prebuild initializer: %w", err) + return commitChanged, xerrors.Errorf("prebuild initializer: %w", err) + } + statusAfter, err := gInit.Status(ctx) + if err != nil { + log.WithError(err).Warn("couldn't run git status - continuing") + } + if statusBefore != nil && statusAfter != nil { + commitChanged = statusBefore.LatestCommit != statusAfter.LatestCommit } err = gInit.UpdateSubmodules(ctx) @@ -173,5 +190,5 @@ func runGitInit(ctx context.Context, gInit *GitInitializer) (err error) { return } }() - return nil + return commitChanged, nil } diff --git a/test/tests/components/ws-manager/prebuild_test.go b/test/tests/components/ws-manager/prebuild_test.go index 8b4ec52127e994..dea267f05026cd 100644 --- a/test/tests/components/ws-manager/prebuild_test.go +++ b/test/tests/components/ws-manager/prebuild_test.go @@ -492,6 +492,162 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) { testEnv.Test(t, f) } +// TestOpenWorkspaceFromOutdatedPrebuild +// - create a prebuild on older commit +// - open a workspace from a later commit with a prebuild initializer +// - make sure the workspace's init task ran (the init task will create a file `incremental.txt` see https://github.com/gitpod-io/test-incremental-workspace/blob/main/.gitpod.yml) +func TestOpenWorkspaceFromOutdatedPrebuild(t *testing.T) { + + f := features.New("prebuild"). + WithLabel("component", "ws-manager"). + Assess("it should open a workspace from with an older prebuild initializer successfully and run the init task", func(_ context.Context, t *testing.T, cfg *envconf.Config) context.Context { + tests := []struct { + Name string + RemoteUri string + CloneTargetForPrebuild string + CloneTargetForWorkspace string + WorkspaceRoot string + CheckoutLocation string + FF []wsmanapi.WorkspaceFeatureFlag + }{ + { + Name: "classic", + RemoteUri: "https://github.com/gitpod-io/test-incremental-workspace", + CloneTargetForPrebuild: "prebuild", + CloneTargetForWorkspace: "main", + CheckoutLocation: "test-incremental-workspace", + WorkspaceRoot: "/workspace/test-incremental-workspace", + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10*len(tests))*time.Minute) + defer cancel() + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + + api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) + t.Cleanup(func() { + api.Done(t) + }) + + // create a prebuild + ws, prebuildStopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error { + req.Type = wsmanapi.WorkspaceType_PREBUILD + req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{ + Name: "GITPOD_TASKS", + Value: `[{ "init": "./init.sh" }]`, + }) + req.Spec.FeatureFlags = test.FF + req.Spec.Initializer = &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Git{ + Git: &csapi.GitInitializer{ + RemoteUri: test.RemoteUri, + TargetMode: csapi.CloneTargetMode_REMOTE_BRANCH, + CloneTaget: test.CloneTargetForPrebuild, + CheckoutLocation: test.CheckoutLocation, + Config: &csapi.GitConfig{}, + }, + }, + } + req.Spec.WorkspaceLocation = test.CheckoutLocation + return nil + })) + if err != nil { + t.Fatalf("cannot launch a workspace: %q", err) + } + defer func() { + // stop workspace in defer function to prevent we forget to stop the workspace + if err := stopWorkspace(t, cfg, prebuildStopWs); err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + }() + prebuildSnapshot, prebuildVSInfo, err := watchStopWorkspaceAndFindSnapshot(ctx, ws.Req.Id, api) + if err != nil { + t.Fatalf("stop workspace and find snapshot error: %v", err) + } + + t.Logf("prebuild snapshot: %s", prebuildSnapshot) + + // launch the workspace on a later commit using this prebuild + ws, stopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error { + req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{ + Name: "GITPOD_TASKS", + Value: `[{ "init": "./init.sh" }]`, + }) + req.Spec.FeatureFlags = test.FF + req.Spec.Initializer = &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Prebuild{ + Prebuild: &csapi.PrebuildInitializer{ + Prebuild: &csapi.SnapshotInitializer{ + Snapshot: prebuildSnapshot, + }, + Git: []*csapi.GitInitializer{ + { + RemoteUri: test.RemoteUri, + TargetMode: csapi.CloneTargetMode_REMOTE_BRANCH, + CloneTaget: test.CloneTargetForWorkspace, + CheckoutLocation: test.CheckoutLocation, + Config: &csapi.GitConfig{}, + }, + }, + }, + }, + } + req.Spec.VolumeSnapshot = prebuildVSInfo + req.Spec.WorkspaceLocation = test.CheckoutLocation + return nil + })) + if err != nil { + t.Fatalf("cannot launch a workspace: %q", err) + } + + defer func() { + // stop workspace in defer function to prevent we forget to stop the workspace + if err := stopWorkspace(t, cfg, stopWs); err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + }() + + rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), + integration.WithInstanceID(ws.Req.Id), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + rsa.Close() + }) + integration.DeferCloser(t, closer) + + var ls agent.ListDirResponse + err = rsa.Call("WorkspaceAgent.ListDir", &agent.ListDirRequest{ + Dir: test.WorkspaceRoot, + }, &ls) + if err != nil { + t.Fatal(err) + } + rsa.Close() + + var found bool + for _, f := range ls.Files { + t.Logf("file: %s", f) + if filepath.Base(f) == "incremental.txt" { + found = true + } + } + if !found { + t.Fatal("did not find incremental.txt") + } + }) + } + return ctx + }). + Feature() + + testEnv.Test(t, f) +} + // TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass // - create a prebuild with small workspace class (20Gi disk) // - create the workspace from prebulid with large workspace class (30Gi disk)