Skip to content

Commit 20b85ed

Browse files
committed
all: add linux-arm64-packet builders
The server has 96 cores and 128 GB of RAM, so run 20 builders on it. This adds the Dockerfile each builder runs in, and the machinery to run 20 copies of the Docker container. Also reduce some logspam in the buildlet's shutdown. (The dashboard/builders.go entry for this builder was added previously in https://golang.org/cl/39851) Fixes golang/go#19929 Change-Id: I8537eee52c002dc9efcadcfb7e78b3a5db07ae44 Reviewed-on: https://go-review.googlesource.com/40392 Reviewed-by: Brad Fitzpatrick <[email protected]>
1 parent 5c1740d commit 20b85ed

File tree

10 files changed

+254
-15
lines changed

10 files changed

+254
-15
lines changed

cmd/buildlet/buildlet.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -198,8 +198,11 @@ func main() {
198198
if !isReverse {
199199
listenForCoordinator()
200200
} else {
201-
err := dialCoordinator() // blocks forever in normal case
202-
log.Fatalf("Error dialing coordinator: %v", err)
201+
if err := dialCoordinator(); err != nil {
202+
log.Fatalf("Error dialing coordinator: %v", err)
203+
}
204+
log.Printf("buildlet reverse mode exiting.")
205+
os.Exit(0)
203206
}
204207
}
205208

cmd/buildlet/reverse.go

+13-4
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,16 @@ func keyForMode(mode string) (string, error) {
4343
}
4444
return key, nil
4545
}
46-
4746
keyPath := filepath.Join(homedir(), ".gobuildkey-"+mode)
47+
if v := os.Getenv("GO_BUILD_KEY_PATH"); v != "" {
48+
keyPath = v
49+
}
4850
key, err := ioutil.ReadFile(keyPath)
51+
if ok, _ := strconv.ParseBool(os.Getenv("GO_BUILD_KEY_DELETE_AFTER_READ")); ok {
52+
os.Remove(keyPath)
53+
}
4954
if err != nil {
50-
if os.IsNotExist(err) && !strings.Contains(*reverse, ",") {
55+
if os.IsNotExist(err) && *reverseType == "" && !strings.Contains(*reverse, ",") {
5156
globalKeyPath := filepath.Join(homedir(), ".gobuildkey")
5257
key, err = ioutil.ReadFile(globalKeyPath)
5358
if err != nil {
@@ -175,10 +180,14 @@ func dialCoordinator() error {
175180

176181
log.Printf("Connected to coordinator; reverse dialing active")
177182
srv := &http.Server{}
178-
err = srv.Serve(revdial.NewListener(bufio.NewReadWriter(
183+
ln := revdial.NewListener(bufio.NewReadWriter(
179184
bufio.NewReader(conn),
180185
bufio.NewWriter(deadlinePerWriteConn{conn, 60 * time.Second}),
181-
)))
186+
))
187+
err = srv.Serve(ln)
188+
if ln.Closed() {
189+
return nil
190+
}
182191
return fmt.Errorf("http.Serve on reverse connection complete: %v", err)
183192
}
184193

cmd/buildlet/stage0/stage0.go

+21-7
Original file line numberDiff line numberDiff line change
@@ -103,27 +103,41 @@ func main() {
103103
cmd.Args = append(cmd.Args, scalewayBuildletArgs()...)
104104
}
105105
if os.Getenv("GO_BUILDER_ENV") == "linux-arm-arm5spacemonkey" {
106-
cmd.Args = append(cmd.Args, reverseBuildletArgs("linux-arm-arm5spacemonkey")...)
106+
cmd.Args = append(cmd.Args, legacyReverseBuildletArgs("linux-arm-arm5spacemonkey")...)
107107
}
108108
switch osArch {
109109
case "linux/s390x":
110110
cmd.Args = append(cmd.Args, "--workdir=/data/golang/workdir")
111-
cmd.Args = append(cmd.Args, reverseBuildletArgs("linux-s390x-ibm")...)
111+
cmd.Args = append(cmd.Args, legacyReverseBuildletArgs("linux-s390x-ibm")...)
112112
case "linux/arm64":
113-
cmd.Args = append(cmd.Args, reverseBuildletArgs("linux-arm64-buildlet")...)
113+
if os.Getenv("GO_BUILDER_ENV") == "host-linux-arm64-packet" {
114+
hostname := os.Getenv("HOSTNAME") // if empty, docker container name is used
115+
cmd.Args = append(cmd.Args,
116+
"--reverse-type=host-linux-arm64-packet",
117+
"--workdir=/workdir",
118+
"--hostname="+hostname,
119+
"--halt=false",
120+
"--reboot=false",
121+
"--coordinator=farmer.golang.org:443",
122+
)
123+
} else {
124+
cmd.Args = append(cmd.Args, legacyReverseBuildletArgs("linux-arm64-buildlet")...)
125+
}
114126
case "linux/ppc64":
115-
cmd.Args = append(cmd.Args, reverseBuildletArgs("linux-ppc64-buildlet")...)
127+
cmd.Args = append(cmd.Args, legacyReverseBuildletArgs("linux-ppc64-buildlet")...)
116128
case "linux/ppc64le":
117-
cmd.Args = append(cmd.Args, reverseBuildletArgs("linux-ppc64le-buildlet")...)
129+
cmd.Args = append(cmd.Args, legacyReverseBuildletArgs("linux-ppc64le-buildlet")...)
118130
case "solaris/amd64":
119-
cmd.Args = append(cmd.Args, reverseBuildletArgs("solaris-amd64-smartosbuildlet")...)
131+
cmd.Args = append(cmd.Args, legacyReverseBuildletArgs("solaris-amd64-smartosbuildlet")...)
120132
}
121133
if err := cmd.Run(); err != nil {
122134
sleepFatalf("Error running buildlet: %v", err)
123135
}
124136
}
125137

126-
func reverseBuildletArgs(builder string) []string {
138+
// legacyReverseBuildletArgs passes builder as the deprecated --reverse flag.
139+
// New code should use --reverse-type instead.
140+
func legacyReverseBuildletArgs(builder string) []string {
127141
return []string{
128142
"--halt=false",
129143
"--reverse=" + builder,
+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright 2017 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// The rundockerbuildlet command loops forever and creates and cleans
6+
// up Docker containers running reverse buildlets. It keeps a fixed
7+
// number of them running at a time. See x/build/env/linux-arm64/packet/README
8+
// for one example user.
9+
package main
10+
11+
import (
12+
"bytes"
13+
"flag"
14+
"fmt"
15+
"io/ioutil"
16+
"log"
17+
"os"
18+
"os/exec"
19+
"path/filepath"
20+
"strings"
21+
"time"
22+
)
23+
24+
var (
25+
image = flag.String("image", "", "docker image to run; required.")
26+
numInst = flag.Int("n", 1, "number of containers to keep running at once")
27+
basename = flag.String("basename", "builder", "prefix before the builder number to use for the container names and host names")
28+
memory = flag.String("memory", "3g", "memory limit flag for docker run")
29+
keyFile = flag.String("key", "/etc/gobuild.key", "go build key file")
30+
)
31+
32+
var buildKey []byte
33+
34+
func main() {
35+
flag.Parse()
36+
37+
key, err := ioutil.ReadFile(*keyFile)
38+
if err != nil {
39+
log.Fatalf("error reading build key from --key=%s: %v", buildKey, err)
40+
}
41+
buildKey = bytes.TrimSpace(key)
42+
43+
if *image == "" {
44+
log.Fatalf("docker --image is required")
45+
}
46+
47+
log.Printf("Started. Will keep %d copies of %s running.", *numInst, *image)
48+
for {
49+
if err := checkFix(); err != nil {
50+
log.Print(err)
51+
}
52+
time.Sleep(time.Second) // TODO: docker wait on the running containers?
53+
}
54+
}
55+
56+
func checkFix() error {
57+
running := map[string]bool{}
58+
59+
out, err := exec.Command("docker", "ps", "-a", "--format", "{{.ID}} {{.Names}} {{.Status}}").Output()
60+
if err != nil {
61+
return fmt.Errorf("error running docker ps: %v", err)
62+
}
63+
// Out is like:
64+
// b1dc9ec2e646 packet14 Up 23 minutes
65+
// eeb458938447 packet11 Exited (0) About a minute ago
66+
// ...
67+
lines := strings.Split(string(out), "\n")
68+
for _, line := range lines {
69+
f := strings.SplitN(line, " ", 3)
70+
if len(f) < 3 {
71+
continue
72+
}
73+
container, name, status := f[0], f[1], f[2]
74+
if !strings.HasPrefix(name, *basename) {
75+
continue
76+
}
77+
if strings.HasPrefix(status, "Exited") {
78+
if out, err := exec.Command("docker", "rm", container).CombinedOutput(); err != nil {
79+
log.Printf("error running docker rm %s: %v, %s", container, err, out)
80+
}
81+
log.Printf("Removed container %s (%s)", container, name)
82+
}
83+
running[name] = strings.HasPrefix(status, "Up")
84+
}
85+
86+
for num := 1; num <= *numInst; num++ {
87+
name := fmt.Sprintf("%s%02d", *basename, num)
88+
if running[name] {
89+
continue
90+
}
91+
log.Printf("Creating %s ...", name)
92+
keyFile := fmt.Sprintf("/tmp/buildkey%s/gobuildkey", num)
93+
if err := os.MkdirAll(filepath.Dir(keyFile), 0700); err != nil {
94+
return err
95+
}
96+
if err := ioutil.WriteFile(keyFile, buildKey, 0600); err != nil {
97+
return err
98+
}
99+
out, err := exec.Command("docker", "run",
100+
"-d",
101+
"--memory="+*memory,
102+
"--name="+name,
103+
"-v", filepath.Dir(keyFile)+":/buildkey/",
104+
"-e", "HOSTNAME="+name,
105+
"--tmpfs=/workdir:rw,exec",
106+
*image).CombinedOutput()
107+
if err != nil {
108+
log.Printf("Error creating %s: %v, %s", name, err, out)
109+
continue
110+
}
111+
log.Printf("Created %v", name)
112+
}
113+
return nil
114+
}

dashboard/builders.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -235,9 +235,9 @@ var Hosts = map[string]*HostConfig{
235235
ReverseAliases: []string{"linux-arm64-buildlet"},
236236
},
237237
"host-linux-arm64-packet": &HostConfig{
238-
Notes: "On 96 core packet.net hosts (Xenial host) in Docker containers (Jesise); run by Go team. See x/build/env/linux-arm64/packet",
238+
Notes: "On 96 core packet.net host (Xenial) in Docker containers (Jessie); run by Go team. See x/build/env/linux-arm64/packet",
239239
IsReverse: true,
240-
ExpectNum: 10,
240+
ExpectNum: 20,
241241
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
242242
},
243243
"host-solaris-amd64": &HostConfig{

env/linux-arm64/packet/Dockerfile

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2017 The Go Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style
3+
# license that can be found in the LICENSE file.
4+
5+
FROM mickaelguene/arm64-debian:jessie
6+
7+
ENV DEBIAN_FRONTEND noninteractive
8+
9+
RUN apt-get update && \
10+
apt-get install --yes \
11+
curl gcc strace ca-certificates \
12+
procps lsof psmisc
13+
14+
RUN mkdir /usr/local/go-bootstrap && \
15+
curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-arm64.tar.gz | \
16+
tar -C /usr/local/go-bootstrap -zxv
17+
18+
ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap
19+
RUN curl -o /usr/local/bin/stage0 https://storage.googleapis.com/go-builder-data/buildlet-stage0.linux-arm64 && \
20+
chmod +x /usr/local/bin/stage0
21+
22+
ENV GO_BUILDER_ENV host-linux-arm64-packet
23+
ENV GO_BUILD_KEY_DELETE_AFTER_READ true
24+
ENV GO_BUILD_KEY_PATH /buildkey/gobuildkey
25+
26+
# Not really, but we're in a container like Kubernetes, and this makes the syscall
27+
# package happy:
28+
ENV IN_KUBERNETES 1
29+
30+
CMD ["/usr/local/bin/stage0"]

env/linux-arm64/packet/README

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
Only needed to create new machines: (skip this step)
2+
3+
https://app.packet.net/
4+
5+
p: see link in email (l:password packet)
6+
7+
Machines:
8+
9+
golang-builder-1:
10+
11+
$ ssh -i ~/keys/id_ed25519_golang1 [email protected]
12+
(key: http://go/golang-ssh-key)
13+
14+
Machine details:
15+
16+
root@golang-builder-1:~# cat /proc/cpuinfo | grep process | wc -l
17+
96
18+
19+
root@golang-builder-1:~# free
20+
total used free shared buff/cache available
21+
Mem: 131971924 526620 131055328 12700 389976 130876088
22+
Swap: 3321056 0 3321056
23+
24+
root@golang-builder-1:~# lsb_release -a
25+
No LSB modules are available.
26+
Distributor ID: Ubuntu
27+
Description: Ubuntu 16.04.1 LTS
28+
Release: 16.04
29+
Codename: xenial
30+
31+
32+
But each machine is setup like:
33+
34+
$ apt-get install docker.io
35+
36+
$ build.sh (with Dockerfile in same directory)
37+
38+
$ GOARCH=arm64 GOOS=linux go install golang.org/x/build/cmd/rundockerbuildlet && \
39+
scp -i ~/keys/id_ed25519_golang1 ~/bin/linux_arm64/rundockerbuildlet [email protected]:/usr/local/bin
40+
41+
$ scp -i ~/keys/id_ed25519_golang1 rundockerbuildlet.service [email protected]:/etc/systemd/user/
42+
43+
$ systemctl enable /etc/systemd/user/rundockerbuildlet.service
44+
$ systemctl start rundockerbuildlet.service

env/linux-arm64/packet/build.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
#
3+
# This is run on the arm64 host, with the Dockerfile in the same directory.
4+
5+
exec docker build -t gobuilder-arm64:1 .
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[Unit]
2+
Description=Run Buildlets in Docker
3+
After=network.target
4+
5+
[Install]
6+
WantedBy=network-online.target
7+
8+
[Service]
9+
Type=simple
10+
ExecStart=/usr/local/bin/rundockerbuildlet -basename=packet -image=gobuilder-arm64:1 -n=20
11+
Restart=always
12+
RestartSec=2
13+
StartLimitInterval=0

revdial/revdial.go

+7
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,13 @@ type Listener struct {
446446
closed bool
447447
}
448448

449+
// Closed reports whether the listener has been closed.
450+
func (ln *Listener) Closed() bool {
451+
ln.mu.Lock()
452+
defer ln.mu.Unlock()
453+
return ln.closed
454+
}
455+
449456
// Accept blocks and returns a new connections, or an error.
450457
func (ln *Listener) Accept() (net.Conn, error) {
451458
c, ok := <-ln.connc

0 commit comments

Comments
 (0)