cmd: merge branch 'dev.link' into master

In the dev.link branch we implemented the new object file format
and (part of) the linker improvements described in
https://golang.org/s/better-linker

The new object file is index-based and provides random access.
The linker maps the object files into read-only memory, and
access symbols on-demand using indices, as opposed to reading
all object files sequentially into the heap with the old format.

The linker carries symbol informations using indices (as opposed
to Symbol data structure). Symbols are created after the
reachability analysis, and only created for reachable symbols.
This reduces the linker's memory usage.

Linking cmd/compile, it creates ~25% fewer Symbols, and reduces
memory usage (inuse_space) by ~15%. (More results from Than.)

Currently, both the old and new object file formats are supported.
The old format is used by default. The new format can be turned
on by using the compiler/assembler/linker's -newobj flag. Note
that the flag needs to be specified consistently to all
compilations, i.e.

go build -gcflags=all=-newobj -asmflags=all=-newobj -ldflags=-newobj

Change-Id: Ia0e35306b5b9b5b19fdc7fa7c602d4ce36fa6abd

Author: cherrymui

src/cmd/asm/internal/flags/flags.go

@@ -23,6 +23,7 @@ var (
 	Dynlink    = flag.Bool("dynlink", false, "support references to Go symbols defined in other shared libraries")
 	AllErrors  = flag.Bool("e", false, "no limit on number of errors reported")
 	SymABIs    = flag.Bool("gensymabis", false, "write symbol ABI information to output file, don't assemble")
+	Newobj     = flag.Bool("newobj", false, "use new object file format")
 )
 
 var (

src/cmd/asm/main.go

@@ -40,18 +40,18 @@ func main() {
 	}
 	ctxt.Flag_dynlink = *flags.Dynlink
 	ctxt.Flag_shared = *flags.Shared || *flags.Dynlink
+	ctxt.Flag_newobj = *flags.Newobj
 	ctxt.Bso = bufio.NewWriter(os.Stdout)
 	defer ctxt.Bso.Flush()
 
 	architecture.Init(ctxt)
 
 	// Create object file, write header.
-	out, err := os.Create(*flags.OutputFile)
+	buf, err := bio.Create(*flags.OutputFile)
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer bio.MustClose(out)
-	buf := bufio.NewWriter(bio.MustWriter(out))
+	defer buf.Close()
 
 	if !*flags.SymABIs {
 		fmt.Fprintf(buf, "go object %s %s %s\n", objabi.GOOS, objabi.GOARCH, objabi.Version)
@@ -83,6 +83,7 @@ func main() {
 		}
 	}
 	if ok && !*flags.SymABIs {
+		ctxt.NumberSyms(true)
 		obj.WriteObjFile(ctxt, buf, "")
 	}
 	if !ok || diag {
@@ -91,9 +92,8 @@ func main() {
 		} else {
 			log.Print("assembly failed")
 		}
-		out.Close()
+		buf.Close()
 		os.Remove(*flags.OutputFile)
 		os.Exit(1)
 	}
-	buf.Flush()
 }

src/cmd/compile/internal/gc/iexport.go

@@ -203,6 +203,7 @@ import (
 	"bufio"
 	"bytes"
 	"cmd/compile/internal/types"
+	"cmd/internal/goobj2"
 	"cmd/internal/src"
 	"encoding/binary"
 	"fmt"
@@ -945,10 +946,12 @@ func (w *exportWriter) string(s string) { w.uint64(w.p.stringOff(s)) }
 
 func (w *exportWriter) varExt(n *Node) {
 	w.linkname(n.Sym)
+	w.symIdx(n.Sym)
 }
 
 func (w *exportWriter) funcExt(n *Node) {
 	w.linkname(n.Sym)
+	w.symIdx(n.Sym)
 
 	// Escape analysis.
 	for _, fs := range types.RecvsParams {
@@ -987,6 +990,22 @@ func (w *exportWriter) linkname(s *types.Sym) {
 	w.string(s.Linkname)
 }
 
+func (w *exportWriter) symIdx(s *types.Sym) {
+	if Ctxt.Flag_newobj {
+		lsym := s.Linksym()
+		if lsym.PkgIdx > goobj2.PkgIdxSelf || (lsym.PkgIdx == goobj2.PkgIdxInvalid && !lsym.Indexed()) || s.Linkname != "" {
+			// Don't export index for non-package symbols, linkname'd symbols,
+			// and symbols without an index. They can only be referenced by
+			// name.
+			w.int64(-1)
+		} else {
+			// For a defined symbol, export its index.
+			// For re-exporting an imported symbol, pass its index through.
+			w.int64(int64(lsym.SymIdx))
+		}
+	}
+}
+
 // Inline bodies.
 
 func (w *exportWriter) stmtList(list Nodes) {

src/cmd/compile/internal/gc/iimport.go

@@ -10,6 +10,7 @@ package gc
 import (
 	"cmd/compile/internal/types"
 	"cmd/internal/bio"
+	"cmd/internal/obj"
 	"cmd/internal/src"
 	"encoding/binary"
 	"fmt"
@@ -651,10 +652,12 @@ func (r *importReader) byte() byte {
 
 func (r *importReader) varExt(n *Node) {
 	r.linkname(n.Sym)
+	r.symIdx(n.Sym)
 }
 
 func (r *importReader) funcExt(n *Node) {
 	r.linkname(n.Sym)
+	r.symIdx(n.Sym)
 
 	// Escape analysis.
 	for _, fs := range types.RecvsParams {
@@ -683,6 +686,20 @@ func (r *importReader) linkname(s *types.Sym) {
 	s.Linkname = r.string()
 }
 
+func (r *importReader) symIdx(s *types.Sym) {
+	if Ctxt.Flag_newobj {
+		lsym := s.Linksym()
+		idx := int32(r.int64())
+		if idx != -1 {
+			if s.Linkname != "" {
+				Fatalf("bad index for linknamed symbol: %v %d\n", lsym, idx)
+			}
+			lsym.SymIdx = idx
+			lsym.Set(obj.AttrIndexed, true)
+		}
+	}
+}
+
 func (r *importReader) doInline(n *Node) {
 	if len(n.Func.Inl.Body) != 0 {
 		Fatalf("%v already has inline body", n)

src/cmd/compile/internal/gc/main.go

@@ -260,6 +260,7 @@ func Main(archInit func(*Arch)) {
 	if supportsDynlink(thearch.LinkArch.Arch) {
 		flag.BoolVar(&flag_shared, "shared", false, "generate code that can be linked into a shared library")
 		flag.BoolVar(&flag_dynlink, "dynlink", false, "support references to Go symbols defined in other shared libraries")
+		flag.BoolVar(&Ctxt.Flag_linkshared, "linkshared", false, "generate code that will be linked against Go shared libraries")
 	}
 	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to `file`")
 	flag.StringVar(&memprofile, "memprofile", "", "write memory profile to `file`")
@@ -274,12 +275,14 @@ func Main(archInit func(*Arch)) {
 	flag.StringVar(&benchfile, "bench", "", "append benchmark times to `file`")
 	flag.BoolVar(&smallFrames, "smallframes", false, "reduce the size limit for stack allocated objects")
 	flag.BoolVar(&Ctxt.UseBASEntries, "dwarfbasentries", Ctxt.UseBASEntries, "use base address selection entries in DWARF")
+	flag.BoolVar(&Ctxt.Flag_newobj, "newobj", false, "use new object file format")
+
 	objabi.Flagparse(usage)
 
 	// Record flags that affect the build result. (And don't
 	// record flags that don't, since that would cause spurious
 	// changes in the binary.)
-	recordFlags("B", "N", "l", "msan", "race", "shared", "dynlink", "dwarflocationlists", "dwarfbasentries", "smallframes")
+	recordFlags("B", "N", "l", "msan", "race", "shared", "dynlink", "dwarflocationlists", "dwarfbasentries", "smallframes", "newobj")
 
 	if smallFrames {
 		maxStackVarSize = 128 * 1024
@@ -746,6 +749,8 @@ func Main(archInit func(*Arch)) {
 
 	// Write object data to disk.
 	timings.Start("be", "dumpobj")
+	dumpdata()
+	Ctxt.NumberSyms(false)
 	dumpobj()
 	if asmhdr != "" {
 		dumpasmhdr()

src/cmd/compile/internal/gc/obj.go

@@ -111,21 +111,7 @@ func dumpCompilerObj(bout *bio.Writer) {
 	dumpexport(bout)
 }
 
-func dumpLinkerObj(bout *bio.Writer) {
-	printObjHeader(bout)
-
-	if len(pragcgobuf) != 0 {
-		// write empty export section; must be before cgo section
-		fmt.Fprintf(bout, "\n$$\n\n$$\n\n")
-		fmt.Fprintf(bout, "\n$$  // cgo\n")
-		if err := json.NewEncoder(bout).Encode(pragcgobuf); err != nil {
-			Fatalf("serializing pragcgobuf: %v", err)
-		}
-		fmt.Fprintf(bout, "\n$$\n\n")
-	}
-
-	fmt.Fprintf(bout, "\n!\n")
-
+func dumpdata() {
 	externs := len(externdcl)
 
 	dumpglobls()
@@ -163,8 +149,24 @@ func dumpLinkerObj(bout *bio.Writer) {
 	}
 
 	addGCLocals()
+}
+
+func dumpLinkerObj(bout *bio.Writer) {
+	printObjHeader(bout)
+
+	if len(pragcgobuf) != 0 {
+		// write empty export section; must be before cgo section
+		fmt.Fprintf(bout, "\n$$\n\n$$\n\n")
+		fmt.Fprintf(bout, "\n$$  // cgo\n")
+		if err := json.NewEncoder(bout).Encode(pragcgobuf); err != nil {
+			Fatalf("serializing pragcgobuf: %v", err)
+		}
+		fmt.Fprintf(bout, "\n$$\n\n")
+	}
+
+	fmt.Fprintf(bout, "\n!\n")
 
-	obj.WriteObjFile(Ctxt, bout.Writer, myimportpath)
+	obj.WriteObjFile(Ctxt, bout, myimportpath)
 }
 
 func addptabs() {

src/cmd/compile/internal/types/sym.go

@@ -80,11 +80,18 @@ func (sym *Sym) Linksym() *obj.LSym {
 	if sym == nil {
 		return nil
 	}
+	initPkg := func(r *obj.LSym) {
+		if sym.Linkname != "" {
+			r.Pkg = "_"
+		} else {
+			r.Pkg = sym.Pkg.Prefix
+		}
+	}
 	if sym.Func() {
 		// This is a function symbol. Mark it as "internal ABI".
-		return Ctxt.LookupABI(sym.LinksymName(), obj.ABIInternal)
+		return Ctxt.LookupABIInit(sym.LinksymName(), obj.ABIInternal, initPkg)
 	}
-	return Ctxt.Lookup(sym.LinksymName())
+	return Ctxt.LookupInit(sym.LinksymName(), initPkg)
 }
 
 // Less reports whether symbol a is ordered before symbol b.

src/cmd/dist/buildtool.go

@@ -54,6 +54,7 @@ var bootstrapDirs = []string{
 	"cmd/internal/gcprog",
 	"cmd/internal/dwarf",
 	"cmd/internal/edit",
+	"cmd/internal/goobj2",
 	"cmd/internal/objabi",
 	"cmd/internal/obj",
 	"cmd/internal/obj/arm",
@@ -72,6 +73,7 @@ var bootstrapDirs = []string{
 	"cmd/link/internal/arm64",
 	"cmd/link/internal/ld",
 	"cmd/link/internal/loadelf",
+	"cmd/link/internal/loader",
 	"cmd/link/internal/loadmacho",
 	"cmd/link/internal/loadpe",
 	"cmd/link/internal/loadxcoff",

src/cmd/dist/test.go

@@ -585,7 +585,7 @@ func (t *tester) registerTests() {
 			},
 		})
 		// Also test a cgo package.
-		if t.cgoEnabled {
+		if t.cgoEnabled && t.internalLink() {
 			t.tests = append(t.tests, distTest{
 				name:    "pie_internal_cgo",
 				heading: "internal linking of -buildmode=pie",

src/cmd/go/alldocs.go

@@ -97,8 +97,8 @@ and test commands:
 	-ldflags '[pattern=]arg list'
 		arguments to pass on each go tool link invocation.
 	-linkshared
-		link against shared libraries previously created with
-		-buildmode=shared.
+		build code that will be linked against shared libraries previously
+		created with -buildmode=shared.
 	-mod mode
 		module download mode to use: readonly or vendor.
 		See 'go help modules' for more.

src/cmd/go/internal/work/build.go

@@ -224,6 +224,7 @@ func buildModeInit() {
 				base.Fatalf("-linkshared not supported on %s\n", platform)
 			}
 			codegenArg = "-dynlink"
+			forcedGcflags = append(forcedGcflags, "-linkshared")
 			// TODO(mwhudson): remove -w when that gets fixed in linker.
 			forcedLdflags = append(forcedLdflags, "-linkshared", "-w")
 		}

src/cmd/go/internal/work/init.go

@@ -1372,7 +1372,13 @@ func PutDefaultFunc(ctxt Context, s *FnState) error {
 	abbrev := DW_ABRV_FUNCTION
 	Uleb128put(ctxt, s.Info, int64(abbrev))
 
-	putattr(ctxt, s.Info, DW_ABRV_FUNCTION, DW_FORM_string, DW_CLS_STRING, int64(len(s.Name)), s.Name)
+	// Expand '"".' to import path.
+	name := s.Name
+	if s.Importpath != "" {
+		name = strings.Replace(name, "\"\".", objabi.PathToPrefix(s.Importpath)+".", -1)
+	}
+
+	putattr(ctxt, s.Info, DW_ABRV_FUNCTION, DW_FORM_string, DW_CLS_STRING, int64(len(name)), name)
 	putattr(ctxt, s.Info, abbrev, DW_FORM_addr, DW_CLS_ADDRESS, 0, s.StartPC)
 	putattr(ctxt, s.Info, abbrev, DW_FORM_addr, DW_CLS_ADDRESS, s.Size, s.StartPC)
 	putattr(ctxt, s.Info, abbrev, DW_FORM_block1, DW_CLS_BLOCK, 1, []byte{DW_OP_call_frame_cfa})

src/cmd/internal/dwarf/dwarf.go

@@ -502,6 +502,14 @@ func (r *objReader) parseObject(prefix []byte) error {
 	}
 	// TODO: extract OS + build ID if/when we need it
 
+	p, err := r.peek(8)
+	if err != nil {
+		return err
+	}
+	if bytes.Equal(p, []byte("\x00go114LD")) {
+		r.readNew()
+		return nil
+	}
 	r.readFull(r.tmp[:8])
 	if !bytes.Equal(r.tmp[:8], []byte("\x00go114ld")) {
 		return r.error(errCorruptObject)