go/types: don't print instance markers for type hashes

This is a port of CL 345891 to go/types.

Change-Id: I5abcb9c9c5110923a743f0c47d9b34b2baabab68
Reviewed-on: https://go-review.googlesource.com/c/go/+/346555
Trust: Robert Findley <[email protected]>
Run-TryBot: Robert Findley <[email protected]>
Reviewed-by: Robert Griesemer <[email protected]>
TryBot-Result: Go Bot <[email protected]>

Author: findleyr

src/go/types/subst.go

@@ -259,28 +259,23 @@ func (subst *subster) typ(typ Type) Type {
 var instanceHashing = 0
 
 func instantiatedHash(typ *Named, targs []Type) string {
+	var buf bytes.Buffer
+
 	assert(instanceHashing == 0)
 	instanceHashing++
-	var buf bytes.Buffer
 	w := newTypeWriter(&buf, nil)
 	w.typeName(typ.obj)
 	w.typeList(targs)
 	instanceHashing--
 
-	// With respect to the represented type, whether a
-	// type is fully expanded or stored as instance
-	// does not matter - they are the same types.
-	// Remove the instanceMarkers printed for instances.
-	res := buf.Bytes()
-	i := 0
-	for _, b := range res {
-		if b != instanceMarker {
-			res[i] = b
-			i++
+	if debug {
+		// there should be no instance markers in type hashes
+		for _, b := range buf.Bytes() {
+			assert(b != instanceMarker)
 		}
 	}
 
-	return string(res[:i])
+	return buf.String()
 }
 
 // typOrNil is like typ but if the argument is nil it is replaced with Typ[Invalid].

src/go/types/typestring.go

@@ -204,7 +204,11 @@ func (w *typeWriter) typ(typ Type) {
 		}
 
 	case *Named:
-		if t.instPos != nil {
+		// Instance markers indicate unexpanded instantiated
+		// types. Write them to aid debugging, but don't write
+		// them when we need an instance hash: whether a type
+		// is fully expanded or not doesn't matter for identity.
+		if instanceHashing == 0 && t.instPos != nil {
 			w.byte(instanceMarker)
 		}
 		w.typeName(t.obj)