ci: add workaround for WSL hanging in CI (#993)

Issue #, if available:
There is a known issue, microsoft/WSL#8529,
where WSL commands can hang. This can cause Windows e2e tests to block
until hitting the 2 hour timeout.

*Description of changes:*
This change adds a workaround to detect the bad state and attempt to
mitigate by killing the WSL service. If the issue cannot be resolved,
the test will only hang for 300 seconds before failing.

*Testing done:*
CI run was successful with 8 WSL shutdown failures.

https://github.com/runfinch/finch/actions/runs/9682445232/job/26715743040

<img width="2051" alt="image"
src="https://github.com/runfinch/finch/assets/55906459/ee582249-8257-48eb-bab3-993150feec80">

- [x] I've reviewed the guidance in CONTRIBUTING.md

*Trade-off analysis*
The trade-off for this approach is the test suite can take longer with
multiple reset VM calls being made. Sample runs which previously took
~15 minutes are up to ~37 minutes with the hanging mitigation; however,
this is down from the 2 hour timeout failure which would occur without
the mitigation.

#### License Acceptance

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

Signed-off-by: Austin Vazquez <[email protected]>

Author: austinvazquez

e2e/vm/vm_darwin_test.go

@@ -96,3 +96,8 @@ var resetDisks = func(_ *option.Option, installed bool) {
 	}
 	gomega.Expect(os.RemoveAll(dataDiskDir)).ShouldNot(gomega.HaveOccurred())
 }
+
+var shutdownWSL = func() error {
+	// no-op on darwin
+	return nil
+}

e2e/vm/vm_test.go

@@ -4,7 +4,6 @@
 package vm
 
 import (
-	"os/exec"
 	"runtime"
 	"time"
 
@@ -29,16 +28,17 @@ var resetVM = func(o *option.Option) {
 		// clean up iptables
 		//nolint:lll // link to explanation
 		// https://docs.rancherdesktop.io/troubleshooting-tips/#q-how-do-i-fix-fata0005-subnet-1040024-overlaps-with-other-one-on-this-address-space-when-running-a-container-using-nerdctl-run
-		gomega.Expect(exec.Command("wsl", "--shutdown").Run()).Should(gomega.BeNil())
+		gomega.Expect(shutdownWSL()).Should(gomega.BeNil())
 	}
 
 	ginkgo.DeferCleanup(func() {
 		writeFile(finchConfigFilePath, origFinchCfg)
 		command.New(o, virtualMachineRootCmd, "stop", "-f").WithoutCheckingExitCode().WithTimeoutInSeconds(20).Run()
 		time.Sleep(1 * time.Second)
 		command.New(o, virtualMachineRootCmd, "remove", "-f").WithoutCheckingExitCode().WithTimeoutInSeconds(10).Run()
+		time.Sleep(1 * time.Second)
 		if runtime.GOOS == "windows" {
-			gomega.Expect(exec.Command("wsl", "--shutdown").Run()).Should(gomega.BeNil())
+			gomega.Expect(shutdownWSL()).Should(gomega.BeNil())
 		}
 		time.Sleep(1 * time.Second)
 		command.New(o, virtualMachineRootCmd, "init").WithoutCheckingExitCode().WithTimeoutInSeconds(160).Run()

e2e/vm/vm_windows_test.go

@@ -7,9 +7,13 @@
 package vm
 
 import (
+	"context"
+	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -60,3 +64,40 @@ var resetDisks = func(_ *option.Option, _ bool) {
 	dataDiskDir := filepath.Join(finchRootDir, ".finch", ".disks")
 	gomega.Expect(os.RemoveAll(dataDiskDir)).ShouldNot(gomega.HaveOccurred())
 }
+
+// shutdownWSL is a wrapper function for "wsl --shutdown".
+//
+// This is a workaround for https://github.com/microsoft/WSL/issues/8529
+//
+// If WSL is suspected of hanging for longer than 180 seconds, then
+// kill the WSL service and retry the shutdown command.
+//
+// This function will at maximum run for 300 seconds before returning
+// context.DeadlineExceeded error.
+var shutdownWSL = func() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
+	defer cancel()
+
+	if err := exec.CommandContext(ctx, "wsl", "--shutdown").Run(); err != nil {
+		ginkgo.GinkgoLogr.Error(err, "WSL shutdown failed", "time", time.Now().Format(time.RFC3339))
+
+		// wsl might be hung, kill the wsl service and try again.
+		// https://github.com/microsoft/WSL/issues/8529
+		killCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+
+		if err := exec.CommandContext(killCtx, "taskkill", "/f", "/im", "wslservice.exe").Run(); err != nil {
+			ginkgo.GinkgoLogr.Error(err, "WSL task kill failed", "time", time.Now().Format(time.RFC3339))
+			return fmt.Errorf("unable to kill wsl service: %w", err)
+		}
+
+		retryCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+
+		if err := exec.CommandContext(retryCtx, "wsl", "--shutdown").Run(); err != nil {
+			return fmt.Errorf("unable to shutdown wsl: %w", err)
+		}
+	}
+
+	return nil
+}