all: split builder config into builder & host configs

Our builders are named of the form "GOOS-GOARCH" or
"GOOS-GOARCH-suffix".

Over time we've grown many builders. This CL doesn't change
that. Builders continue to be named and operate as before.

Previously the build configuration file (dashboard/builders.go) made
each builder type ("linux-amd64-race", etc) define how to create a
host running a buildlet of that type, even though many builders had
identical host configs. For example, these builders all share the same
host type (a Kubernetes container):

   linux-amd64
   linux-amd64-race
   linux-386
   linux-386-387

And these are the same host type (a GCE VM):

   windows-amd64-gce
   windows-amd64-race
   windows-386-gce

This CL creates a new concept of a "hostType" which defines how
the buildlet is created (Kube, GCE, Reverse, and how), and then each
builder itself references a host type.

Users never see the hostType. (except perhaps in gomote list output)
But they at least never need to care about them.

Reverse buildlets now can only be one hostType at a time, which
simplifies things. We were no longer using multiple roles per machine
once moving to VMs for OS X.

gomote continues to operate as it did previously but its underlying
protocol changed and clients will need to be updated. As a new
feature, gomote now has a new flag to let you reuse a buildlet host
connection for different builder rules if they share the same
underlying host type. But users can ignore that.

This CL is a long-standing TODO (previously attempted and aborted) and
will make many things easier and faster, including the linux-arm
cross-compilation effort, and keeping pre-warmed buildlets of VM types
ready to go.

Updates golang/go#17104

Change-Id: Iad8387f48680424a8441e878a2f4762bf79ea4d2
Reviewed-on: https://go-review.googlesource.com/29551
Reviewed-by: Matthew Dempsky <[email protected]>

Author: bradfitz

buildlet/gce.go

@@ -74,12 +74,15 @@ type VMOpts struct {
 
 // StartNewVM boots a new VM on GCE and returns a buildlet client
 // configured to speak to it.
-func StartNewVM(ts oauth2.TokenSource, instName, builderType string, opts VMOpts) (*Client, error) {
+func StartNewVM(ts oauth2.TokenSource, instName, hostType string, opts VMOpts) (*Client, error) {
 	computeService, _ := compute.New(oauth2.NewClient(oauth2.NoContext, ts))
 
-	conf, ok := dashboard.Builders[builderType]
+	hconf, ok := dashboard.Hosts[hostType]
 	if !ok {
-		return nil, fmt.Errorf("invalid builder type %q", builderType)
+		return nil, fmt.Errorf("invalid host type %q", hostType)
+	}
+	if !hconf.IsGCE() {
+		return nil, fmt.Errorf("host type %q is not a GCE host type", hostType)
 	}
 
 	zone := opts.Zone
@@ -96,9 +99,9 @@ func StartNewVM(ts oauth2.TokenSource, instName, builderType string, opts VMOpts
 	usePreempt := false
 Try:
 	prefix := "https://www.googleapis.com/compute/v1/projects/" + projectID
-	machType := prefix + "/zones/" + zone + "/machineTypes/" + conf.MachineType()
+	machType := prefix + "/zones/" + zone + "/machineTypes/" + hconf.MachineType()
 	diskType := "https://www.googleapis.com/compute/v1/projects/" + projectID + "/zones/" + zone + "/diskTypes/pd-ssd"
-	if conf.RegularDisk {
+	if hconf.RegularDisk {
 		diskType = "" // a spinning disk
 	}
 
@@ -126,7 +129,7 @@ Try:
 				Type:       "PERSISTENT",
 				InitializeParams: &compute.AttachedDiskInitializeParams{
 					DiskName:    instName,
-					SourceImage: "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + conf.VMImage,
+					SourceImage: "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + hconf.VMImage,
 					DiskType:    diskType,
 				},
 			},
@@ -159,8 +162,8 @@ Try:
 	// which the VMs are configured to download at boot and run.
 	// This lets us/ update the buildlet more easily than
 	// rebuilding the whole VM image.
-	addMeta("buildlet-binary-url", conf.BuildletBinaryURL(buildenv.ByProjectID(opts.ProjectID)))
-	addMeta("builder-type", builderType)
+	addMeta("buildlet-binary-url", hconf.BuildletBinaryURL(buildenv.ByProjectID(opts.ProjectID)))
+	addMeta("buildlet-host-type", hostType)
 	if !opts.TLS.IsZero() {
 		addMeta("tls-cert", opts.TLS.CertPEM)
 		addMeta("tls-key", opts.TLS.KeyPEM)
@@ -304,7 +307,7 @@ type VM struct {
 	Name   string
 	IPPort string
 	TLS    KeyPair
-	Type   string
+	Type   string // buildlet type
 }
 
 // ListVMs lists all VMs.
@@ -329,13 +332,13 @@ func ListVMs(ts oauth2.TokenSource, proj, zone string) ([]VM, error) {
 				meta[it.Key] = *it.Value
 			}
 		}
-		builderType := meta["builder-type"]
-		if builderType == "" {
+		hostType := meta["buildlet-host-type"]
+		if hostType == "" {
 			continue
 		}
 		vm := VM{
 			Name: inst.Name,
-			Type: builderType,
+			Type: hostType,
 			TLS: KeyPair{
 				CertPEM: meta["tls-cert"],
 				KeyPEM:  meta["tls-key"],

buildlet/kube.go

@@ -67,10 +67,10 @@ type PodOpts struct {
 
 // StartPod creates a new pod on a Kubernetes cluster and returns a buildlet client
 // configured to speak to it.
-func StartPod(ctx context.Context, kubeClient *kubernetes.Client, podName, builderType string, opts PodOpts) (*Client, error) {
-	conf, ok := dashboard.Builders[builderType]
+func StartPod(ctx context.Context, kubeClient *kubernetes.Client, podName, hostType string, opts PodOpts) (*Client, error) {
+	conf, ok := dashboard.Hosts[hostType]
 	if !ok || conf.KubeImage == "" {
-		return nil, fmt.Errorf("invalid builder type %q", builderType)
+		return nil, fmt.Errorf("invalid builder type %q", hostType)
 	}
 	pod := &api.Pod{
 		TypeMeta: api.TypeMeta{
@@ -81,7 +81,7 @@ func StartPod(ctx context.Context, kubeClient *kubernetes.Client, podName, build
 			Name: podName,
 			Labels: map[string]string{
 				"name": podName,
-				"type": builderType,
+				"type": hostType,
 				"role": "buildlet",
 			},
 			Annotations: map[string]string{},
@@ -132,7 +132,7 @@ func StartPod(ctx context.Context, kubeClient *kubernetes.Client, podName, build
 	// This lets us/ update the buildlet more easily than
 	// rebuilding the whole pod image.
 	addEnv("META_BUILDLET_BINARY_URL", conf.BuildletBinaryURL(buildenv.ByProjectID(opts.ProjectID)))
-	addEnv("META_BUILDER_TYPE", builderType)
+	addEnv("META_BUILDLET_HOST_TYPE", hostType)
 	if !opts.TLS.IsZero() {
 		addEnv("META_TLS_CERT", opts.TLS.CertPEM)
 		addEnv("META_TLS_KEY", opts.TLS.KeyPEM)

buildlet/remote.go

@@ -58,17 +58,25 @@ func (cc *CoordinatorClient) client() (*http.Client, error) {
 	return cc.hc, nil
 }
 
-// CreateBuildlet creates a new buildlet of the given type on cc.
+// CreateBuildlet creates a new buildlet of the given builder type on
+// cc.
+//
+// This takes a builderType (instead of a hostType), but the
+// returned buildlet can be used as any builder that has the same
+// underlying buildlet type. For instance, a linux-amd64 buildlet can
+// act as either linux-amd64 or linux-386-387.
+//
 // It may expire at any time.
 // To release it, call Client.Destroy.
-func (cc *CoordinatorClient) CreateBuildlet(buildletType string) (*Client, error) {
+func (cc *CoordinatorClient) CreateBuildlet(builderType string) (*Client, error) {
 	hc, err := cc.client()
 	if err != nil {
 		return nil, err
 	}
 	ipPort, _ := cc.instance().TLSHostPort() // must succeed if client did
 	form := url.Values{
-		"type": {buildletType},
+		"version":     {"20160922"}, // checked by cmd/coordinator/remote.go
+		"builderType": {builderType},
 	}
 	req, _ := http.NewRequest("POST",
 		"https://"+ipPort+"/buildlet/create",
@@ -100,10 +108,11 @@ func (cc *CoordinatorClient) CreateBuildlet(buildletType string) (*Client, error
 }
 
 type RemoteBuildlet struct {
-	Type    string // "openbsd-386"
-	Name    string // "buildlet-adg-openbsd-386-2"
-	Created time.Time
-	Expires time.Time
+	HostType    string // "host-linux-kubestd"
+	BuilderType string // "linux-386-387"
+	Name        string // "buildlet-adg-openbsd-386-2"
+	Created     time.Time
+	Expires     time.Time
 }
 
 func (cc *CoordinatorClient) RemoteBuildlets() ([]RemoteBuildlet, error) {
@@ -113,7 +122,6 @@ func (cc *CoordinatorClient) RemoteBuildlets() ([]RemoteBuildlet, error) {
 	}
 	ipPort, _ := cc.instance().TLSHostPort() // must succeed if client did
 	req, _ := http.NewRequest("GET", "https://"+ipPort+"/buildlet/list", nil)
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 	req.SetBasicAuth(cc.Auth.Username, cc.Auth.Password)
 	res, err := hc.Do(req)
 	if err != nil {

cmd/buildlet/Makefile

@@ -58,16 +58,16 @@ buildlet.solaris-amd64: FORCE
 
 dev-buildlet.linux-arm: FORCE
 	go install golang.org/x/build/cmd/upload
-	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/$@
+	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/buildlet.linux-arm
 
 dev-buildlet.linux-amd64: FORCE
 	go install golang.org/x/build/cmd/upload
-	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/$@
+	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/buildlet.linux-amd64
 
 dev-buildlet.windows-amd64: FORCE
 	go install golang.org/x/build/cmd/upload
-	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/$@
+	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/buildlet.windows-amd64
 
 dev-buildlet.plan9-386: FORCE
 	go install golang.org/x/build/cmd/upload
-	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/$@
+	upload --verbose --osarch=$@ --file=go:golang.org/x/build/cmd/buildlet --public dev-go-builder-data/buildlet.plan9-386

cmd/buildlet/buildlet.go

@@ -509,7 +509,7 @@ func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
 		defer res.Body.Close()
 		if res.StatusCode != http.StatusOK {
 			log.Printf("writetgz: failed to fetch tgz URL %s: status=%v", urlStr, res.Status)
-			http.Error(w, fmt.Sprintf("fetching provided url: %s", res.Status), http.StatusInternalServerError)
+			http.Error(w, fmt.Sprintf("writetgz: fetching provided URL %q: %s", urlStr, res.Status), http.StatusInternalServerError)
 			return
 		}
 		tgz = res.Body
@@ -736,7 +736,14 @@ func handleExec(w http.ResponseWriter, r *http.Request) {
 		f.Flush()
 	}
 
-	env := append(baseEnv(), r.PostForm["env"]...)
+	goarch := "amd64" // unless we find otherwise
+	for _, pair := range r.PostForm["env"] {
+		if hasPrefixFold(pair, "GOARCH=") {
+			goarch = pair[len("GOARCH="):]
+		}
+	}
+
+	env := append(baseEnv(goarch), r.PostForm["env"]...)
 	env = envutil.Dedup(runtime.GOOS == "windows", env)
 	env = setPathEnv(env, r.PostForm["path"], *workDir)
 
@@ -869,21 +876,17 @@ func pathSeparator() string {
 	}
 }
 
-func baseEnv() []string {
+func baseEnv(goarch string) []string {
 	if runtime.GOOS == "windows" {
-		return windowsBaseEnv()
+		return windowsBaseEnv(goarch)
 	}
 	return os.Environ()
 }
 
-func windowsBaseEnv() (e []string) {
+func windowsBaseEnv(goarch string) (e []string) {
 	e = append(e, "GOBUILDEXIT=1") // exit all.bat with completion status
-	btype, err := metadata.InstanceAttributeValue("builder-type")
-	if err != nil {
-		log.Fatalf("Failed to get builder-type: %v", err)
-		return nil
-	}
-	is64 := strings.HasPrefix(btype, "windows-amd64")
+
+	is64 := goarch != "386"
 	for _, pair := range os.Environ() {
 		const pathEq = "PATH="
 		if hasPrefixFold(pair, pathEq) {
@@ -1239,9 +1242,9 @@ func makeBSDFilesystemFast() {
 		log.Printf("Not on GCE; not remounting root filesystem.")
 		return
 	}
-	btype, err := metadata.InstanceAttributeValue("builder-type")
+	btype, err := metadata.InstanceAttributeValue("buildlet-type")
 	if _, ok := err.(metadata.NotDefinedError); ok && len(btype) == 0 {
-		log.Printf("Not remounting root filesystem due to missing builder-type metadata.")
+		log.Printf("Not remounting root filesystem due to missing buildlet-type metadata.")
 		return
 	}
 	if err != nil {