internal/task: start x repo tagging tasks

First step: list all our projects/repositories, and select those that
have a go.mod labeling them as golang.org/x as candidates to release as
the next version, or v0.1.0 for untagged repositories.

For golang/go#48523.

Change-Id: Ice92319a0726daf3bf5f94581582d8802640dffc
Reviewed-on: https://go-review.googlesource.com/c/build/+/425088
Reviewed-by: Dmitri Shuralyov <[email protected]>
Auto-Submit: Heschi Kreinick <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Run-TryBot: Heschi Kreinick <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

Author: heschi

Diff for: gerrit/gerrit.go

@@ -12,11 +12,11 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"sort"
@@ -116,11 +116,17 @@ type urlValues url.Values
 
 func (urlValues) isDoArg() {}
 
+// respBodyRaw returns the body of the response. If set, dst is ignored.
+type respBodyRaw struct{ rc *io.ReadCloser }
+
+func (respBodyRaw) isDoArg() {}
+
 func (c *Client) do(ctx context.Context, dst interface{}, method, path string, opts ...doArg) error {
 	var arg url.Values
-	var body io.Reader
+	var requestBody io.Reader
 	var contentType string
 	var wantStatus = http.StatusOK
+	var responseBody *io.ReadCloser
 	for _, opt := range opts {
 		switch opt := opt.(type) {
 		case wantResStatus:
@@ -130,13 +136,15 @@ func (c *Client) do(ctx context.Context, dst interface{}, method, path string, o
 			if err != nil {
 				return err
 			}
-			body = bytes.NewReader(b)
+			requestBody = bytes.NewReader(b)
 			contentType = "application/json"
 		case reqBodyRaw:
-			body = opt.r
+			requestBody = opt.r
 			contentType = "application/octet-stream"
 		case urlValues:
 			arg = url.Values(opt)
+		case respBodyRaw:
+			responseBody = opt.rc
 		default:
 			panic(fmt.Sprintf("internal error; unsupported type %T", opt))
 		}
@@ -148,12 +156,11 @@ func (c *Client) do(ctx context.Context, dst interface{}, method, path string, o
 	if _, ok := c.auth.(noAuth); ok {
 		slashA = ""
 	}
-	var err error
 	u := c.url + slashA + path
 	if arg != nil {
 		u += "?" + arg.Encode()
 	}
-	req, err := http.NewRequestWithContext(ctx, method, u, body)
+	req, err := http.NewRequestWithContext(ctx, method, u, requestBody)
 	if err != nil {
 		return err
 	}
@@ -167,16 +174,27 @@ func (c *Client) do(ctx context.Context, dst interface{}, method, path string, o
 	if err != nil {
 		return err
 	}
-	defer res.Body.Close()
+	defer func() {
+		if responseBody != nil && *responseBody != nil {
+			// We've handed off the body to the user.
+			return
+		}
+		res.Body.Close()
+	}()
 
 	if res.StatusCode != wantStatus {
-		body, err := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
+		body, err := io.ReadAll(io.LimitReader(res.Body, 4<<10))
 		return &HTTPError{res, body, err}
 	}
 
+	if responseBody != nil {
+		*responseBody = res.Body
+		return nil
+	}
+
 	if dst == nil {
 		// Drain the response body, return an error if it's anything but empty.
-		body, err := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
+		body, err := io.ReadAll(io.LimitReader(res.Body, 4<<10))
 		if err != nil || len(body) != 0 {
 			return &HTTPError{res, body, err}
 		}
@@ -711,7 +729,7 @@ type ProjectInfo struct {
 
 // ListProjects returns the server's active projects.
 //
-// The returned slice is sorted by project ID and excludes the "All-Projects" project.
+// The returned slice is sorted by project ID and excludes the "All-Projects" and "All-Users" projects.
 //
 // See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#list-projects
 func (c *Client) ListProjects(ctx context.Context) ([]ProjectInfo, error) {
@@ -722,12 +740,15 @@ func (c *Client) ListProjects(ctx context.Context) ([]ProjectInfo, error) {
 	}
 	var ret []ProjectInfo
 	for name, pi := range res {
-		if name == "All-Projects" {
+		if name == "All-Projects" || name == "All-Users" {
 			continue
 		}
 		if pi.State != "ACTIVE" {
 			continue
 		}
+		// https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#project-info:
+		// "name not set if returned in a map where the project name is used as map key"
+		pi.Name = name
 		ret = append(ret, pi)
 	}
 	sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID })
@@ -828,6 +849,26 @@ func (c *Client) GetBranch(ctx context.Context, project, branch string) (BranchI
 	return res, err
 }
 
+// GetFileContent gets a file's contents at a particular commit.
+//
+// See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-content-from-commit.
+func (c *Client) GetFileContent(ctx context.Context, project, commit, path string) (io.ReadCloser, error) {
+	var body io.ReadCloser
+	err := c.do(ctx, nil, "GET", fmt.Sprintf("/projects/%s/commits/%s/files/%s/content", project, commit, url.QueryEscape(path)), respBodyRaw{&body})
+	if err != nil {
+		return nil, err
+	}
+	return readCloser{
+		Reader: base64.NewDecoder(base64.StdEncoding, body),
+		Closer: body,
+	}, nil
+}
+
+type readCloser struct {
+	io.Reader
+	io.Closer
+}
+
 // WebLinkInfo is information about a web link.
 // See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#web-link-info
 type WebLinkInfo struct {

Diff for: go.mod

@@ -46,6 +46,7 @@ require (
 	go4.org v0.0.0-20180809161055-417644f6feb5
 	golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70
 	golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d
+	golang.org/x/mod v0.5.1
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
 	golang.org/x/perf v0.0.0-20220913151710-7c6e287988f3

Diff for: go.sum

@@ -946,7 +946,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

Diff for: internal/relui/buildrelease_test.go

@@ -562,6 +562,7 @@ type fakeGerrit struct {
 	changesCreated int
 	createdTags    map[string]string
 	wantReviewers  []string
+	task.GerritClient
 }
 
 func (g *fakeGerrit) CreateAutoSubmitChange(ctx context.Context, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) {

Diff for: internal/task/gerrit.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"strings"
 
 	"golang.org/x/build/gerrit"
@@ -27,6 +28,10 @@ type GerritClient interface {
 	ListTags(ctx context.Context, project string) ([]string, error)
 	// ReadBranchHead returns the head of a branch in project.
 	ReadBranchHead(ctx context.Context, project, branch string) (string, error)
+	// ListProjects lists all the projects on the server.
+	ListProjects(ctx context.Context) ([]string, error)
+	// ReadFile reads a file from project at the specified commit.
+	ReadFile(ctx context.Context, project, commit, file string) ([]byte, error)
 }
 
 type RealGerritClient struct {
@@ -149,6 +154,27 @@ func (c *RealGerritClient) ReadBranchHead(ctx context.Context, project, branch s
 	return branchInfo.Revision, nil
 }
 
+func (c *RealGerritClient) ListProjects(ctx context.Context) ([]string, error) {
+	projects, err := c.Client.ListProjects(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, p := range projects {
+		names = append(names, p.Name)
+	}
+	return names, nil
+}
+
+func (c *RealGerritClient) ReadFile(ctx context.Context, project, commit, file string) ([]byte, error) {
+	body, err := c.Client.GetFileContent(ctx, project, commit, file)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+	return io.ReadAll(body)
+}
+
 // ChangeLink returns a link to the review page for the CL with the specified
 // change ID. The change ID must be in the project~cl# form.
 func ChangeLink(changeID string) string {

Diff for: internal/task/tagx.go

@@ -0,0 +1,101 @@
+package task
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"golang.org/x/build/gerrit"
+	wf "golang.org/x/build/internal/workflow"
+	"golang.org/x/mod/modfile"
+	"golang.org/x/mod/semver"
+)
+
+type TagXReposTasks struct {
+	Gerrit GerritClient
+}
+
+type TagRepo struct {
+	Name string
+	Next string
+	Deps []string
+}
+
+func (x *TagXReposTasks) SelectRepos(ctx *wf.TaskContext) ([]*TagRepo, error) {
+	projects, err := x.Gerrit.ListProjects(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx.Printf("Examining repositories %v", projects)
+	var repos []*TagRepo
+	for _, p := range projects {
+		repo, err := x.readRepo(ctx, p)
+		if err != nil {
+			return nil, err
+		}
+		if repo != nil {
+			repos = append(repos, repo)
+		}
+	}
+	return repos, nil
+}
+
+func (x *TagXReposTasks) readRepo(ctx *wf.TaskContext, project string) (*TagRepo, error) {
+	head, err := x.Gerrit.ReadBranchHead(ctx, project, "master")
+	if errors.Is(err, gerrit.ErrResourceNotExist) {
+		ctx.Printf("ignoring %v: no master branch: %v", project, err)
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	gomod, err := x.Gerrit.ReadFile(ctx, project, head, "go.mod")
+	if errors.Is(err, gerrit.ErrResourceNotExist) {
+		ctx.Printf("ignoring %v: no go.mod: %v", project, err)
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	mf, err := modfile.ParseLax("go.mod", gomod, nil)
+	if err != nil {
+		return nil, err
+	}
+	if !strings.HasPrefix(mf.Module.Mod.Path, "golang.org/x") {
+		ctx.Printf("ignoring %v: not golang.org/x", project)
+		return nil, nil
+	}
+
+	tags, err := x.Gerrit.ListTags(ctx, project)
+	if err != nil {
+		return nil, err
+	}
+	highestRelease := ""
+	for _, tag := range tags {
+		if semver.IsValid(tag) && semver.Prerelease(tag) == "" &&
+			(highestRelease == "" || semver.Compare(highestRelease, tag) < 0) {
+			highestRelease = tag
+		}
+	}
+	nextTag := "v0.1.0"
+	if highestRelease != "" {
+		var err error
+		nextTag, err = nextVersion(highestRelease)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't pick next version for %v: %v", project, err)
+		}
+	}
+
+	result := &TagRepo{
+		Name: project,
+		Next: nextTag,
+	}
+	for _, req := range mf.Require {
+		if strings.HasPrefix(req.Mod.Path, "golang.org/x") {
+			result.Deps = append(result.Deps, req.Mod.Path)
+		}
+	}
+	return result, nil
+}

Diff for: internal/task/tagx_test.go

@@ -0,0 +1,35 @@
+package task
+
+import (
+	"context"
+	"flag"
+	"testing"
+
+	"golang.org/x/build/gerrit"
+	"golang.org/x/build/internal/workflow"
+)
+
+var flagRunTagXTest = flag.Bool("run-tagx-test", false, "run tag x/ repo test, which is read-only and safe. Must have a Gerrit cookie in gitcookies.")
+
+func TestSelectReposLive(t *testing.T) {
+	if !*flagRunTagXTest {
+		t.Skip("Not enabled by flags")
+	}
+
+	tasks := &TagXReposTasks{
+		Gerrit: &RealGerritClient{
+			Client: gerrit.NewClient("https://go-review.googlesource.com", gerrit.GitCookiesAuth()),
+		},
+	}
+	ctx := &workflow.TaskContext{
+		Context: context.Background(),
+		Logger:  &testLogger{t},
+	}
+	repos, err := tasks.SelectRepos(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range repos {
+		t.Logf("%#v", r)
+	}
+}