feat: support cloning over SSH via private key auth

README.md

@@ -288,6 +288,8 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de
 | `--git-clone-single-branch` | `GIT_CLONE_SINGLE_BRANCH` |  | Clone only a single branch of the Git repository. |
 | `--git-username` | `GIT_USERNAME` |  | The username to use for Git authentication. This is optional. |
 | `--git-password` | `GIT_PASSWORD` |  | The password to use for Git authentication. This is optional. |
+| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` |  | Path to a SSH private key to be used for Git authentication. |
+| `--git-ssh-known-hosts-base64` | `GIT_SSH_KNOWN_HOSTS_BASE64` |  | Base64-encoded content of a known hosts file. If not specified, host keys will be scanned and logged, but not checked. |
 | `--git-http-proxy-url` | `GIT_HTTP_PROXY_URL` |  | The URL for the HTTP proxy. This is optional. |
 | `--workspace-folder` | `WORKSPACE_FOLDER` |  | The path to the workspace folder that will be built. This is optional. |
 | `--ssl-cert-base64` | `SSL_CERT_BASE64` |  | The content of an SSL cert file. This is useful for self-signed certificates. |

envbuilder.go

@@ -45,6 +45,7 @@ import (
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/sirupsen/logrus"
@@ -156,6 +157,34 @@ func Run(ctx context.Context, options Options) error {
 		}
 	}
 
+	gitURLParsed, err := url.Parse(options.GitURL)
+	if err != nil {
+		return fmt.Errorf("invalid git URL: %w", err)
+	}
+	// If we're cloning over SSH, we need a known_hosts file.
+	if gitURLParsed.Scheme == "ssh" {
+		var knownHostsContent []byte
+		if options.GitSSHKnownHostsBase64 != "" {
+			if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil {
+				return fmt.Errorf("invalid known_hosts content: %w", err)
+			} else {
+				knownHostsContent = kh
+			}
+		} else {
+			kh, err := GenerateKnownHosts(options.Logger, gitURLParsed)
+			if err != nil {
+				return fmt.Errorf("invalid known_hosts content: %w", err)
+			} else {
+				knownHostsContent = kh
+			}
+		}
+		knownHostsPath := filepath.Join(MagicDir, "known_hosts")
+		if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil {
+			return fmt.Errorf("write known_hosts file: %w", err)
+		}
+		_ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
+	}
+
 	var fallbackErr error
 	var cloned bool
 	if options.GitURL != "" {
@@ -201,6 +230,15 @@ func Run(ctx context.Context, options Options) error {
 				Username: options.GitUsername,
 				Password: options.GitPassword,
 			}
+		} else if options.GitSSHPrivateKeyPath != "" {
+			signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath)
+			if err != nil {
+				return xerrors.Errorf("read private key: %w", err)
+			}
+			cloneOpts.RepoAuth = &gitssh.PublicKeys{
+				User:   "git",
+				Signer: signer,
+			}
 		}
 		if options.GitHTTPProxyURL != "" {
 			cloneOpts.ProxyOptions = transport.ProxyOptions{

git.go

@@ -1,11 +1,19 @@
 package envbuilder
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
+	"net"
 	"net/url"
+	"os"
+	"strconv"
+	"strings"
 
+	"github.com/coder/coder/v2/codersdk"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -14,6 +22,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/storage/filesystem"
+	gossh "golang.org/x/crypto/ssh"
 )
 
 type CloneRepoOptions struct {
@@ -113,3 +122,55 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) {
 	}
 	return true, nil
 }
+
+func ReadPrivateKey(path string) (gossh.Signer, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open private key file: %w", err)
+	}
+	var buf bytes.Buffer
+	if _, err := io.Copy(&buf, f); err != nil {
+		return nil, fmt.Errorf("read private key file: %w", err)
+	}
+	k, err := gossh.ParsePrivateKey(buf.Bytes())
+	if err != nil {
+		return nil, fmt.Errorf("parse private key file: %w", err)
+	}
+	return k, nil
+}
+
+// GenerateKnownHosts dials the server located at gitURL and fetches the SSH
+// public keys returned in a format accepted by known_hosts.
+func GenerateKnownHosts(log LoggerFunc, gitURL *url.URL) ([]byte, error) {
+	var buf bytes.Buffer
+	conf := &gossh.ClientConfig{
+		// Accept and record all host keys
+		HostKeyCallback: func(dialAddr string, addr net.Addr, key gossh.PublicKey) error {
+			h := strings.Split(dialAddr, ":")[0]
+			k64 := base64.StdEncoding.EncodeToString(key.Marshal())
+			log(codersdk.LogLevelInfo, "ssh keyscan: %s %s %s", h, key.Type(), k64)
+			buf.WriteString(fmt.Sprintf("%s %s %s\n", h, key.Type(), k64))
+			return nil
+		},
+	}
+	dialAddr := hostPort(gitURL)
+	client, err := gossh.Dial("tcp", dialAddr, conf)
+	if err != nil {
+		// The dial may fail due to no authentication methods, but this is fine.
+		if netErr, ok := err.(net.Error); ok {
+			return nil, fmt.Errorf("keyscan %s: %w", dialAddr, netErr)
+		}
+		// If it's not a net.Error then we will assume we were successful.
+	} else {
+		_ = client.Close()
+	}
+	return buf.Bytes(), nil
+}
+
+func hostPort(u *url.URL) string {
+	p := 22 // assume default SSH port
+	if _p, err := strconv.Atoi(u.Port()); err == nil {
+		p = _p
+	}
+	return fmt.Sprintf("%s:%d", u.Host, p)
+}

git_test.go

@@ -2,20 +2,25 @@ package envbuilder_test
 
 import (
 	"context"
+	"crypto/ed25519"
 	"fmt"
 	"io"
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path/filepath"
 	"regexp"
 	"testing"
 
 	"github.com/coder/envbuilder"
 	"github.com/coder/envbuilder/testutil/gittest"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-billy/v5/osfs"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/stretchr/testify/require"
+	gossh "golang.org/x/crypto/ssh"
 )
 
 func TestCloneRepo(t *testing.T) {
@@ -159,6 +164,101 @@ func TestCloneRepo(t *testing.T) {
 	}
 }
 
+func TestCloneRepoSSH(t *testing.T) {
+
+	// nolint: paralleltest // t.Setenv
+	t.Run("PrivateKeyOK", func(t *testing.T) {
+		t.Skip("TODO: need to figure out how to properly add advertised refs")
+		// TODO: Can't we use a memfs here?
+		tmpDir := t.TempDir()
+		srvFS := osfs.New(tmpDir, osfs.WithChrootOS())
+
+		signer := randKeygen(t)
+		_ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!"))
+		tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey())
+		gitURL := tr.String()
+		clientFS := memfs.New()
+
+		cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{
+			Path:    "/workspace",
+			RepoURL: gitURL,
+			Storage: clientFS,
+			RepoAuth: &gitssh.PublicKeys{
+				User:   "",
+				Signer: signer,
+				HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{
+					HostKeyCallback: gossh.InsecureIgnoreHostKey(), // TODO: known_hosts
+				},
+			},
+		})
+		require.NoError(t, err) // TODO: error: repository not found
+		require.True(t, cloned)
+
+		readme := mustRead(t, clientFS, "/workspace/README.md")
+		require.Equal(t, "Hello, world!", readme)
+		gitConfig := mustRead(t, clientFS, "/workspace/.git/config")
+		// Ensure we do not modify the git URL that folks pass in.
+		require.Regexp(t, fmt.Sprintf(`(?m)^\s+url\s+=\s+%s\s*$`, regexp.QuoteMeta(gitURL)), gitConfig)
+	})
+
+	// nolint: paralleltest // t.Setenv
+	t.Run("PrivateKeyError", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		srvFS := osfs.New(tmpDir, osfs.WithChrootOS())
+
+		signer := randKeygen(t)
+		anotherSigner := randKeygen(t)
+		_ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!"))
+		tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey())
+		gitURL := tr.String()
+		clientFS := memfs.New()
+
+		cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{
+			Path:    "/workspace",
+			RepoURL: gitURL,
+			Storage: clientFS,
+			RepoAuth: &gitssh.PublicKeys{
+				User:   "",
+				Signer: anotherSigner,
+				HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{
+					HostKeyCallback: gossh.InsecureIgnoreHostKey(), // TODO: known_hosts
+				},
+			},
+		})
+		require.ErrorContains(t, err, "handshake failed")
+		require.False(t, cloned)
+	})
+
+	// nolint: paralleltest // t.Setenv
+	t.Run("PrivateKeyHostKeyUnknown", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		srvFS := osfs.New(tmpDir, osfs.WithChrootOS())
+
+		knownHostsPath := filepath.Join(tmpDir, "known_hosts")
+		require.NoError(t, os.WriteFile(knownHostsPath, []byte{}, 0o600))
+		t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
+
+		signer := randKeygen(t)
+		anotherSigner := randKeygen(t)
+		_ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!"))
+		tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey())
+		gitURL := tr.String()
+		clientFS := memfs.New()
+
+		cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{
+			Path:    "/workspace",
+			RepoURL: gitURL,
+			Storage: clientFS,
+			RepoAuth: &gitssh.PublicKeys{
+				User:   "",
+				Signer: anotherSigner,
+			},
+		})
+		require.ErrorContains(t, err, "key is unknown")
+		require.False(t, cloned)
+	})
+}
+
 func mustRead(t *testing.T, fs billy.Filesystem, path string) string {
 	t.Helper()
 	f, err := fs.OpenFile(path, os.O_RDONLY, 0644)
@@ -167,3 +267,13 @@ func mustRead(t *testing.T, fs billy.Filesystem, path string) string {
 	require.NoError(t, err)
 	return string(content)
 }
+
+// generates a random ed25519 private key
+func randKeygen(t *testing.T) gossh.Signer {
+	t.Helper()
+	_, key, err := ed25519.GenerateKey(nil)
+	require.NoError(t, err)
+	signer, err := gossh.NewSignerFromKey(key)
+	require.NoError(t, err)
+	return signer
+}

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/docker/cli v26.1.0+incompatible
 	github.com/docker/docker v23.0.8+incompatible
 	github.com/fatih/color v1.16.0
+	github.com/gliderlabs/ssh v0.3.7
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/google/go-containerregistry v0.15.2
@@ -36,6 +37,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
+	golang.org/x/crypto v0.21.0
 	golang.org/x/sync v0.7.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
 )
@@ -70,6 +72,7 @@ require (
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/akutz/memconn v0.1.0 // indirect
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.20.3 // indirect
@@ -261,7 +264,6 @@ require (
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect
 	go4.org/netipx v0.0.0-20230728180743-ad4cb58a6516 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect
 	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.23.0 // indirect