internal/relui,internal/task: move useful structs to task

For whatever reason I've decided to write the tagging workflow in the
task package, so I need the test helpers here. I also need the website
structs to find the latest Go release later.

For golang/go#48523.

Change-Id: I92a1430bd27f0aa90c9fccefa12fb057e7e7f864
Reviewed-on: https://go-review.googlesource.com/c/build/+/425090
Reviewed-by: Dmitri Shuralyov <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Run-TryBot: Heschi Kreinick <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Auto-Submit: Heschi Kreinick <[email protected]>

Author: heschi

Diff for: cmd/relui/main.go

@@ -175,7 +175,7 @@ func main() {
 		ScratchURL:       *scratchFilesBase,
 		ServingURL:       *servingFilesBase,
 		DownloadURL:      *edgeCacheURL,
-		PublishFile: func(f *relui.WebsiteFile) error {
+		PublishFile: func(f *task.WebsiteFile) error {
 			return publishFile(*websiteUploadURL, userPassAuth, f)
 		},
 		ApproveAction: relui.ApproveActionDep(dbPool),
@@ -228,7 +228,7 @@ func key(masterKey, principal string) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func publishFile(uploadURL string, auth buildlet.UserPass, f *relui.WebsiteFile) error {
+func publishFile(uploadURL string, auth buildlet.UserPass, f *task.WebsiteFile) error {
 	req, err := json.Marshal(f)
 	if err != nil {
 		return err

Diff for: internal/relui/buildrelease_test.go

@@ -66,7 +66,7 @@ type releaseTestDeps struct {
 	versionTasks   *task.VersionTasks
 	buildTasks     *BuildReleaseTasks
 	milestoneTasks *task.MilestoneTasks
-	publishedFiles map[string]*WebsiteFile
+	publishedFiles map[string]*task.WebsiteFile
 	outputListener func(taskName string, output interface{})
 }
 
@@ -115,8 +115,8 @@ func newReleaseTestDeps(t *testing.T, wantVersion string) *releaseTestDeps {
 
 	// Set up the fake website to publish to.
 	var filesMu sync.Mutex
-	files := map[string]*WebsiteFile{}
-	publishFile := func(f *WebsiteFile) error {
+	files := map[string]*task.WebsiteFile{}
+	publishFile := func(f *task.WebsiteFile) error {
 		filesMu.Lock()
 		defer filesMu.Unlock()
 		files[strings.TrimPrefix(f.Filename, wantVersion+".")] = f
@@ -197,20 +197,20 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) {
 	}
 
 	dlURL, files := deps.buildTasks.DownloadURL, deps.publishedFiles
-	checkTGZ(t, dlURL, files, "src.tar.gz", &WebsiteFile{
+	checkTGZ(t, dlURL, files, "src.tar.gz", &task.WebsiteFile{
 		OS:   "",
 		Arch: "",
 		Kind: "source",
 	}, map[string]string{
 		"go/VERSION":       wantVersion,
 		"go/src/make.bash": makeScript,
 	})
-	checkContents(t, dlURL, files, "windows-amd64.msi", &WebsiteFile{
+	checkContents(t, dlURL, files, "windows-amd64.msi", &task.WebsiteFile{
 		OS:   "windows",
 		Arch: "amd64",
 		Kind: "installer",
 	}, "I'm an MSI!\n")
-	checkTGZ(t, dlURL, files, "linux-amd64.tar.gz", &WebsiteFile{
+	checkTGZ(t, dlURL, files, "linux-amd64.tar.gz", &task.WebsiteFile{
 		OS:   "linux",
 		Arch: "amd64",
 		Kind: "archive",
@@ -219,23 +219,23 @@ func testRelease(t *testing.T, wantVersion string, kind task.ReleaseKind) {
 		"go/tool/something_orother/compile": "",
 		"go/pkg/something_orother/race.a":   "",
 	})
-	checkZip(t, dlURL, files, "windows-arm64.zip", &WebsiteFile{
+	checkZip(t, dlURL, files, "windows-arm64.zip", &task.WebsiteFile{
 		OS:   "windows",
 		Arch: "arm64",
 		Kind: "archive",
 	}, map[string]string{
 		"go/VERSION":                        wantVersion,
 		"go/tool/something_orother/compile": "",
 	})
-	checkTGZ(t, dlURL, files, "linux-armv6l.tar.gz", &WebsiteFile{
+	checkTGZ(t, dlURL, files, "linux-armv6l.tar.gz", &task.WebsiteFile{
 		OS:   "linux",
 		Arch: "armv6l",
 		Kind: "archive",
 	}, map[string]string{
 		"go/VERSION":                        wantVersion,
 		"go/tool/something_orother/compile": "",
 	})
-	checkContents(t, dlURL, files, "darwin-amd64.pkg", &WebsiteFile{
+	checkContents(t, dlURL, files, "darwin-amd64.pkg", &task.WebsiteFile{
 		OS:   "darwin",
 		Arch: "amd64",
 		Kind: "installer",
@@ -312,7 +312,7 @@ func testSecurity(t *testing.T, mergeFixes bool) {
 		runToFailure(t, deps.ctx, w, "Check branch state matches source archive", &verboseListener{t, deps.outputListener})
 		return
 	}
-	checkTGZ(t, deps.buildTasks.DownloadURL, deps.publishedFiles, "src.tar.gz", &WebsiteFile{
+	checkTGZ(t, deps.buildTasks.DownloadURL, deps.publishedFiles, "src.tar.gz", &task.WebsiteFile{
 		OS:   "",
 		Arch: "",
 		Kind: "source",
@@ -465,13 +465,13 @@ func serveTarball(pathMatch string, files map[string]string, w http.ResponseWrit
 	}
 }
 
-func checkFile(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, check func(*testing.T, []byte)) {
+func checkFile(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, check func(*testing.T, []byte)) {
 	t.Run(filename, func(t *testing.T) {
 		f, ok := files[filename]
 		if !ok {
 			t.Fatalf("file %q not published", filename)
 		}
-		if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" {
+		if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(task.WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" {
 			t.Errorf("file metadata mismatch (-want +got):\n%v", diff)
 		}
 		resp, err := http.Get(dlURL + "/" + f.Filename)
@@ -486,15 +486,15 @@ func checkFile(t *testing.T, dlURL string, files map[string]*WebsiteFile, filena
 	})
 }
 
-func checkContents(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents string) {
+func checkContents(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents string) {
 	checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) {
 		if got, want := string(b), contents; got != want {
 			t.Errorf("%v contains %q, want %q", filename, got, want)
 		}
 	})
 }
 
-func checkTGZ(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) {
+func checkTGZ(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents map[string]string) {
 	checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) {
 		gzr, err := gzip.NewReader(bytes.NewReader(b))
 		if err != nil {
@@ -528,7 +528,7 @@ func checkTGZ(t *testing.T, dlURL string, files map[string]*WebsiteFile, filenam
 	})
 }
 
-func checkZip(t *testing.T, dlURL string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) {
+func checkZip(t *testing.T, dlURL string, files map[string]*task.WebsiteFile, filename string, meta *task.WebsiteFile, contents map[string]string) {
 	checkFile(t, dlURL, files, filename, meta, func(t *testing.T, b []byte) {
 		zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
 		if err != nil {

Diff for: internal/relui/workflows.go

@@ -550,7 +550,7 @@ type BuildReleaseTasks struct {
 	GCSClient              *storage.Client
 	ScratchURL, ServingURL string
 	DownloadURL            string
-	PublishFile            func(*WebsiteFile) error
+	PublishFile            func(*task.WebsiteFile) error
 	CreateBuildlet         func(context.Context, string) (buildlet.RemoteClient, error)
 	ApproveAction          func(*wf.TaskContext) error
 }
@@ -705,8 +705,8 @@ func (b *BuildReleaseTasks) runBuildStep(
 			return artifact{}, err
 		}
 		defer client.Close()
-		w := &logWriter{logger: ctx}
-		go w.run(ctx)
+		w := &task.LogWriter{Logger: ctx}
+		go w.Run(ctx)
 		step = &task.BuildletStep{
 			Target:      target,
 			Buildlet:    client,
@@ -812,62 +812,6 @@ func (w *sizeWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-type logWriter struct {
-	flushTicker *time.Ticker
-
-	mu     sync.Mutex
-	buf    []byte
-	logger wf.Logger
-}
-
-func (w *logWriter) Write(b []byte) (int, error) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-
-	w.buf = append(w.buf, b...)
-	if len(w.buf) > 1<<20 {
-		w.flushLocked(false)
-		w.flushTicker.Reset(10 * time.Second)
-	}
-	return len(b), nil
-}
-
-func (w *logWriter) flush(force bool) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	w.flushLocked(force)
-}
-
-func (w *logWriter) flushLocked(force bool) {
-	if len(w.buf) == 0 {
-		return
-	}
-	log, rest := w.buf, []byte(nil)
-	if !force {
-		nl := bytes.LastIndexByte(w.buf, '\n')
-		if nl == -1 {
-			return
-		}
-		log, rest = w.buf[:nl], w.buf[nl+1:]
-	}
-	w.logger.Printf("\n%s", string(log))
-	w.buf = append([]byte(nil), rest...) // don't leak
-}
-
-func (w *logWriter) run(ctx context.Context) {
-	w.flushTicker = time.NewTicker(10 * time.Second)
-	defer w.flushTicker.Stop()
-	for {
-		select {
-		case <-w.flushTicker.C:
-			w.flush(false)
-		case <-ctx.Done():
-			w.flush(true)
-			return
-		}
-	}
-}
-
 func (tasks *BuildReleaseTasks) startSigningCommand(ctx *wf.TaskContext, version string) (string, error) {
 	args := fmt.Sprintf("--relui_staging=%q", tasks.ScratchURL+"/"+signingStagingDir(ctx, version))
 	ctx.Printf("run signer with " + args)
@@ -1099,7 +1043,7 @@ func uploadArtifact(scratchFS, servingFS fs.FS, a artifact) error {
 // The version string uses the same format as Go tags. For example, "go1.19rc1".
 func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version string, artifacts []artifact) (publishedVersion string, _ error) {
 	for _, a := range artifacts {
-		f := &WebsiteFile{
+		f := &task.WebsiteFile{
 			Filename:       a.Filename,
 			Version:        version,
 			ChecksumSHA256: a.SHA256,
@@ -1127,15 +1071,3 @@ func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version st
 	ctx.Printf("Published %v artifacts for %v", len(artifacts), version)
 	return version, nil
 }
-
-// WebsiteFile represents a file on the go.dev downloads page.
-// It should be kept in sync with the download code in x/website/internal/dl.
-type WebsiteFile struct {
-	Filename       string `json:"filename"`
-	OS             string `json:"os"`
-	Arch           string `json:"arch"`
-	Version        string `json:"version"`
-	ChecksumSHA256 string `json:"sha256"`
-	Size           int64  `json:"size"`
-	Kind           string `json:"kind"` // "archive", "installer", "source"
-}

Diff for: internal/task/task.go

@@ -6,9 +6,12 @@
 package task
 
 import (
+	"bytes"
+	"context"
+	"sync"
 	"time"
 
-	"golang.org/x/build/internal/workflow"
+	wf "golang.org/x/build/internal/workflow"
 )
 
 // CommunicationTasks combines communication tasks together.
@@ -22,7 +25,7 @@ var AwaitDivisor int = 1
 // AwaitCondition calls the condition function every period until it returns
 // true to indicate success, or an error. If the condition succeeds,
 // AwaitCondition returns its result.
-func AwaitCondition[T any](ctx *workflow.TaskContext, period time.Duration, condition func() (T, bool, error)) (T, error) {
+func AwaitCondition[T any](ctx *wf.TaskContext, period time.Duration, condition func() (T, bool, error)) (T, error) {
 	pollTimer := time.NewTicker(period / time.Duration(AwaitDivisor))
 	defer pollTimer.Stop()
 	for {
@@ -39,3 +42,89 @@ func AwaitCondition[T any](ctx *workflow.TaskContext, period time.Duration, cond
 		}
 	}
 }
+
+// LogWriter is an io.Writer that writes to a workflow task's log, flushing
+// its buffer periodically to avoid too many writes.
+type LogWriter struct {
+	Logger wf.Logger
+
+	flushTicker *time.Ticker
+
+	mu  sync.Mutex
+	buf []byte
+}
+
+func (w *LogWriter) Write(b []byte) (int, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	w.buf = append(w.buf, b...)
+	if len(w.buf) > 1<<20 {
+		w.flushLocked(false)
+		w.flushTicker.Reset(10 * time.Second)
+	}
+	return len(b), nil
+}
+
+func (w *LogWriter) flush(force bool) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.flushLocked(force)
+}
+
+func (w *LogWriter) flushLocked(force bool) {
+	if len(w.buf) == 0 {
+		return
+	}
+	log, rest := w.buf, []byte(nil)
+	if !force {
+		nl := bytes.LastIndexByte(w.buf, '\n')
+		if nl == -1 {
+			return
+		}
+		log, rest = w.buf[:nl], w.buf[nl+1:]
+	}
+	w.Logger.Printf("\n%s", string(log))
+	w.buf = append([]byte(nil), rest...) // don't leak
+}
+
+func (w *LogWriter) Run(ctx context.Context) {
+	w.flushTicker = time.NewTicker(10 * time.Second)
+	defer w.flushTicker.Stop()
+	for {
+		select {
+		case <-w.flushTicker.C:
+			w.flush(false)
+		case <-ctx.Done():
+			w.flush(true)
+			return
+		}
+	}
+}
+
+// WebsiteFile represents a file on the go.dev downloads page.
+// It should be kept in sync with the download code in x/website/internal/dl.
+type WebsiteFile struct {
+	Filename       string `json:"filename"`
+	OS             string `json:"os"`
+	Arch           string `json:"arch"`
+	Version        string `json:"version"`
+	ChecksumSHA256 string `json:"sha256"`
+	Size           int64  `json:"size"`
+	Kind           string `json:"kind"` // "archive", "installer", "source"
+}
+
+func (f WebsiteFile) GOARCH() string {
+	if f.OS == "linux" && f.Arch == "armv6l" {
+		return "arm"
+	}
+	return f.Arch
+}
+
+type WebsiteRelease struct {
+	Version        string        `json:"version"`
+	Stable         bool          `json:"stable"`
+	Files          []WebsiteFile `json:"files"`
+	Visible        bool          `json:"-"` // show files on page load
+	SplitPortTable bool          `json:"-"` // whether files should be split by primary/other ports.
+}