cmd/go: reroute vcs-test.golang.org HTTPS requests to the test-local server

After this CL, the only test requests that should still reach
vcs-test.golang.org are for Subversion repos, which are not yet handled.

The interceptor implementation should also allow us to redirect other
servers (such as gopkg.in) fairly easily in a followup change if
desired.

For golang#27494.

Change-Id: I8cb85f3a7edbbf0492662ff5cfa779fb9b407136
Reviewed-on: https://go-review.googlesource.com/c/go/+/427254
Auto-Submit: Bryan Mills <[email protected]>
Run-TryBot: Bryan Mills <[email protected]>
Reviewed-by: Russ Cox <[email protected]>

Author: Bryan C. Mills

src/cmd/go/go_test.go

@@ -35,6 +35,7 @@ import (
 	"cmd/go/internal/search"
 	"cmd/go/internal/vcs"
 	"cmd/go/internal/vcweb/vcstest"
+	"cmd/go/internal/web"
 	"cmd/go/internal/work"
 	"cmd/internal/sys"
 
@@ -132,9 +133,21 @@ func TestMain(m *testing.M) {
 			}
 		}
 
-		if vcsTest := os.Getenv("TESTGO_VCSTEST_URL"); vcsTest != "" {
-			vcs.VCSTestRepoURL = vcsTest
+		if vcsTestHost := os.Getenv("TESTGO_VCSTEST_HOST"); vcsTestHost != "" {
+			vcs.VCSTestRepoURL = "http://" + vcsTestHost
 			vcs.VCSTestHosts = vcstest.Hosts
+			vcsTestTLSHost := os.Getenv("TESTGO_VCSTEST_TLS_HOST")
+			vcsTestClient, err := vcstest.TLSClient(os.Getenv("TESTGO_VCSTEST_CERT"))
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "loading certificates from $TESTGO_VCSTEST_CERT: %v", err)
+			}
+			var interceptors []web.Interceptor
+			for _, host := range vcstest.Hosts {
+				interceptors = append(interceptors,
+					web.Interceptor{Scheme: "http", FromHost: host, ToHost: vcsTestHost},
+					web.Interceptor{Scheme: "https", FromHost: host, ToHost: vcsTestTLSHost, Client: vcsTestClient})
+			}
+			web.EnableTestHooks(interceptors)
 		}
 
 		cmdgo.Main()

src/cmd/go/internal/auth/auth.go

@@ -10,7 +10,10 @@ import "net/http"
 // AddCredentials fills in the user's credentials for req, if any.
 // The return value reports whether any matching credentials were found.
 func AddCredentials(req *http.Request) (added bool) {
-	host := req.URL.Hostname()
+	host := req.Host
+	if host == "" {
+		host = req.URL.Hostname()
+	}
 
 	// TODO(golang.org/issue/26232): Support arbitrary user-provided credentials.
 	netrcOnce.Do(readNetrc)

src/cmd/go/internal/vcs/vcs.go

@@ -1208,7 +1208,9 @@ func interceptVCSTest(repo string, vcs *Cmd, security web.SecurityMode) (repoURL
 		return "", false
 	}
 	if vcs == vcsMod {
-		return "", false // Will be implemented in CL 427254.
+		// Since the "mod" protocol is implemented internally,
+		// requests will be intercepted at a lower level (in cmd/go/internal/web).
+		return "", false
 	}
 	if vcs == vcsSvn {
 		return "", false // Will be implemented in CL 427914.

src/cmd/go/internal/vcweb/auth.go

@@ -0,0 +1,108 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vcweb
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+)
+
+// authHandler serves requests only if the Basic Auth data sent with the request
+// matches the contents of a ".access" file in the requested directory.
+//
+// For each request, the handler looks for a file named ".access" and parses it
+// as a JSON-serialized accessToken. If the credentials from the request match
+// the accessToken, the file is served normally; otherwise, it is rejected with
+// the StatusCode and Message provided by the token.
+type authHandler struct{}
+
+type accessToken struct {
+	Username, Password string
+	StatusCode         int // defaults to 401.
+	Message            string
+}
+
+func (h *authHandler) Available() bool { return true }
+
+func (h *authHandler) Handler(dir string, env []string, logger *log.Logger) (http.Handler, error) {
+	fs := http.Dir(dir)
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		urlPath := req.URL.Path
+		if urlPath != "" && strings.HasPrefix(path.Base(urlPath), ".") {
+			http.Error(w, "filename contains leading dot", http.StatusBadRequest)
+			return
+		}
+
+		f, err := fs.Open(urlPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				http.NotFound(w, req)
+			} else {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+			}
+			return
+		}
+
+		accessDir := urlPath
+		if fi, err := f.Stat(); err == nil && !fi.IsDir() {
+			accessDir = path.Dir(urlPath)
+		}
+		f.Close()
+
+		var accessFile http.File
+		for {
+			var err error
+			accessFile, err = fs.Open(path.Join(accessDir, ".access"))
+			if err == nil {
+				break
+			}
+
+			if !os.IsNotExist(err) {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			if accessDir == "." {
+				http.Error(w, "failed to locate access file", http.StatusInternalServerError)
+				return
+			}
+			accessDir = path.Dir(accessDir)
+		}
+
+		data, err := ioutil.ReadAll(accessFile)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var token accessToken
+		if err := json.Unmarshal(data, &token); err != nil {
+			logger.Print(err)
+			http.Error(w, "malformed access file", http.StatusInternalServerError)
+			return
+		}
+		if username, password, ok := req.BasicAuth(); !ok || username != token.Username || password != token.Password {
+			code := token.StatusCode
+			if code == 0 {
+				code = http.StatusUnauthorized
+			}
+			if code == http.StatusUnauthorized {
+				w.Header().Add("WWW-Authenticate", fmt.Sprintf("basic realm=%s", accessDir))
+			}
+			http.Error(w, token.Message, code)
+			return
+		}
+
+		http.FileServer(fs).ServeHTTP(w, req)
+	})
+
+	return handler, nil
+}

src/cmd/go/internal/vcweb/insecure.go

@@ -0,0 +1,42 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vcweb
+
+import (
+	"log"
+	"net/http"
+)
+
+// insecureHandler redirects requests to the same host and path but using the
+// "http" scheme instead of "https".
+type insecureHandler struct{}
+
+func (h *insecureHandler) Available() bool { return true }
+
+func (h *insecureHandler) Handler(dir string, env []string, logger *log.Logger) (http.Handler, error) {
+	// The insecure-redirect handler implementation doesn't depend or dir or env.
+	//
+	// The only effect of the directory is to determine which prefix the caller
+	// will strip from the request before passing it on to this handler.
+	return h, nil
+}
+
+func (h *insecureHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	if req.Host == "" && req.URL.Host == "" {
+		http.Error(w, "no Host provided in request", http.StatusBadRequest)
+		return
+	}
+
+	// Note that if the handler is wrapped with http.StripPrefix, the prefix
+	// will remain stripped in the redirected URL, preventing redirect loops
+	// if the scheme is already "http".
+
+	u := *req.URL
+	u.Scheme = "http"
+	u.User = nil
+	u.Host = req.Host
+
+	http.Redirect(w, req, u.String(), http.StatusFound)
+}

src/cmd/go/internal/vcweb/script.go

@@ -21,6 +21,9 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"golang.org/x/mod/module"
+	"golang.org/x/mod/zip"
 )
 
 // newScriptEngine returns a script engine augmented with commands for
@@ -38,6 +41,7 @@ func newScriptEngine() *script.Engine {
 	cmds["git"] = script.Program("git", interrupt, gracePeriod)
 	cmds["hg"] = script.Program("hg", interrupt, gracePeriod)
 	cmds["handle"] = scriptHandle()
+	cmds["modzip"] = scriptModzip()
 	cmds["svn"] = script.Program("svn", interrupt, gracePeriod)
 	cmds["unquote"] = scriptUnquote()
 
@@ -280,6 +284,40 @@ func scriptHandle() script.Cmd {
 		})
 }
 
+func scriptModzip() script.Cmd {
+	return script.Command(
+		script.CmdUsage{
+			Summary: "create a Go module zip file from a directory",
+			Args:    "zipfile path@version dir",
+		},
+		func(st *script.State, args ...string) (wait script.WaitFunc, err error) {
+			if len(args) != 3 {
+				return nil, script.ErrUsage
+			}
+			zipPath := st.Path(args[0])
+			mPath, version, ok := strings.Cut(args[1], "@")
+			if !ok {
+				return nil, script.ErrUsage
+			}
+			dir := st.Path(args[2])
+
+			if err := os.MkdirAll(filepath.Dir(zipPath), 0755); err != nil {
+				return nil, err
+			}
+			f, err := os.Create(zipPath)
+			if err != nil {
+				return nil, err
+			}
+			defer func() {
+				if closeErr := f.Close(); err == nil {
+					err = closeErr
+				}
+			}()
+
+			return nil, zip.CreateFromDir(f, module.Version{Path: mPath, Version: version}, dir)
+		})
+}
+
 func scriptUnquote() script.Cmd {
 	return script.Command(
 		script.CmdUsage{

src/cmd/go/internal/vcweb/vcstest/vcstest.go

@@ -9,11 +9,17 @@ package vcstest
 import (
 	"cmd/go/internal/vcs"
 	"cmd/go/internal/vcweb"
+	"cmd/go/internal/web"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"internal/testenv"
 	"io"
 	"log"
+	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"path/filepath"
 	"testing"
@@ -26,6 +32,7 @@ var Hosts = []string{
 type Server struct {
 	workDir string
 	HTTP    *httptest.Server
+	HTTPS   *httptest.Server
 }
 
 // NewServer returns a new test-local vcweb server that serves VCS requests
@@ -58,15 +65,45 @@ func NewServer() (srv *Server, err error) {
 	}
 
 	srvHTTP := httptest.NewServer(handler)
+	httpURL, err := url.Parse(srvHTTP.URL)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			srvHTTP.Close()
+		}
+	}()
+
+	srvHTTPS := httptest.NewTLSServer(handler)
+	httpsURL, err := url.Parse(srvHTTPS.URL)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			srvHTTPS.Close()
+		}
+	}()
 
 	srv = &Server{
 		workDir: workDir,
 		HTTP:    srvHTTP,
+		HTTPS:   srvHTTPS,
 	}
 	vcs.VCSTestRepoURL = srv.HTTP.URL
 	vcs.VCSTestHosts = Hosts
 
+	var interceptors []web.Interceptor
+	for _, host := range Hosts {
+		interceptors = append(interceptors,
+			web.Interceptor{Scheme: "http", FromHost: host, ToHost: httpURL.Host, Client: srv.HTTP.Client()},
+			web.Interceptor{Scheme: "https", FromHost: host, ToHost: httpsURL.Host, Client: srv.HTTPS.Client()})
+	}
+	web.EnableTestHooks(interceptors)
+
 	fmt.Fprintln(os.Stderr, "vcs-test.golang.org rerouted to "+srv.HTTP.URL)
+	fmt.Fprintln(os.Stderr, "https://vcs-test.golang.org rerouted to "+srv.HTTPS.URL)
 
 	return srv, nil
 }
@@ -77,7 +114,45 @@ func (srv *Server) Close() error {
 	}
 	vcs.VCSTestRepoURL = ""
 	vcs.VCSTestHosts = nil
+	web.DisableTestHooks()
 
 	srv.HTTP.Close()
+	srv.HTTPS.Close()
 	return os.RemoveAll(srv.workDir)
 }
+
+func (srv *Server) WriteCertificateFile() (string, error) {
+	b := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: srv.HTTPS.Certificate().Raw,
+	})
+
+	filename := filepath.Join(srv.workDir, "cert.pem")
+	if err := os.WriteFile(filename, b, 0644); err != nil {
+		return "", err
+	}
+	return filename, nil
+}
+
+// TLSClient returns an http.Client that can talk to the httptest.Server
+// whose certificate is written to the given file path.
+func TLSClient(certFile string) (*http.Client, error) {
+	client := &http.Client{
+		Transport: http.DefaultTransport.(*http.Transport).Clone(),
+	}
+
+	pemBytes, err := os.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+
+	certpool := x509.NewCertPool()
+	if !certpool.AppendCertsFromPEM(pemBytes) {
+		return nil, fmt.Errorf("no certificates found in %s", certFile)
+	}
+	client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{
+		RootCAs: certpool,
+	}
+
+	return client, nil
+}