Merge pull request #795 from thedadams/cred-helper-releases

feat: download binaries for cred helpers

Author: thedadams

Diff for: pkg/cli/credential.go

@@ -58,7 +58,7 @@ func (c *Credential) Run(cmd *cobra.Command, _ []string) error {
 		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)
 	}
 
-	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil {
+	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {
 		return err
 	}
 

Diff for: pkg/cli/credential_delete.go

@@ -40,7 +40,7 @@ func (c *Delete) Run(cmd *cobra.Command, args []string) error {
 		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)
 	}
 
-	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil {
+	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {
 		return err
 	}
 

Diff for: pkg/cli/credential_show.go

@@ -42,7 +42,7 @@ func (c *Show) Run(cmd *cobra.Command, args []string) error {
 		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)
 	}
 
-	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil {
+	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {
 		return err
 	}
 

Diff for: pkg/credentials/util.go

@@ -5,15 +5,13 @@ import (
 )
 
 type CredentialHelperDirs struct {
-	RevisionFile, LastCheckedFile, BinDir, RepoDir, HelperDir string
+	RevisionFile, LastCheckedFile, BinDir string
 }
 
 func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs {
 	return CredentialHelperDirs{
 		RevisionFile:    filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "revision"),
 		LastCheckedFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "last-checked"),
 		BinDir:          filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "bin"),
-		RepoDir:         filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "repo"),
-		HelperDir:       filepath.Join(cacheDir, "repos", "gptscript-credential-helpers"),
 	}
 }

Diff for: pkg/engine/engine.go

@@ -22,7 +22,7 @@ type Model interface {
 type RuntimeManager interface {
 	GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error)
 	EnsureCredentialHelpers(ctx context.Context) error
-	SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error
+	SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error
 }
 
 type Engine struct {

Diff for: pkg/gptscript/gptscript.go

@@ -99,7 +99,7 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) {
 		opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir())
 	}
 
-	if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg, opts.Env); err != nil {
+	if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil {
 		return nil, err
 	}
 

Diff for: pkg/repos/get.go

@@ -8,22 +8,21 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/BurntSushi/locker"
 	"github.com/gptscript-ai/gptscript/pkg/config"
 	"github.com/gptscript-ai/gptscript/pkg/credentials"
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
-	"github.com/gptscript-ai/gptscript/pkg/loader/github"
 	"github.com/gptscript-ai/gptscript/pkg/repos/git"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang"
 	"github.com/gptscript-ai/gptscript/pkg/types"
 )
 
-const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-helpers"
-
 type Runtime interface {
 	ID() string
 	Supports(tool types.Tool, cmd []string) bool
@@ -68,7 +67,6 @@ type credHelperConfig struct {
 	lock        sync.Mutex
 	initialized bool
 	cliCfg      *config.CLIConfig
-	env         []string
 }
 
 func New(cacheDir string, runtimes ...Runtime) *Manager {
@@ -90,7 +88,7 @@ func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error {
 	defer m.credHelperConfig.lock.Unlock()
 
 	if !m.credHelperConfig.initialized {
-		if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg, m.credHelperConfig.env); err != nil {
+		if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg); err != nil {
 			return err
 		}
 		m.credHelperConfig.initialized = true
@@ -99,27 +97,28 @@ func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error {
 	return nil
 }
 
-func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig, env []string) error {
+func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig) error {
 	m.credHelperConfig = &credHelperConfig{
 		cliCfg: cliCfg,
-		env:    env,
 	}
 	return nil
 }
 
-func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error {
+func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error {
 	var (
-		helperName = cliCfg.CredentialsStore
-		suffix     string
+		helperName       = cliCfg.CredentialsStore
+		distInfo, suffix string
 	)
-	if helperName == "wincred" {
-		suffix = ".exe"
-	}
-
-	// The file helper is built-in and does not need to be compiled.
+	// The file helper is built-in and does not need to be downloaded.
 	if helperName == "file" {
 		return nil
 	}
+	switch helperName {
+	case "wincred":
+		suffix = ".exe"
+	default:
+		distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH)
+	}
 
 	locker.Lock("gptscript-credential-helpers")
 	defer locker.Unlock("gptscript-credential-helpers")
@@ -137,13 +136,7 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co
 		}
 	}
 
-	// Load the credential helpers repo information.
-	_, _, repo, _, err := github.Load(ctx, nil, credentialHelpersRepo)
-	if err != nil {
-		return err
-	}
-
-	if err := os.MkdirAll(m.credHelperDirs.HelperDir, 0755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(m.credHelperDirs.LastCheckedFile), 0755); err != nil {
 		return err
 	}
 
@@ -152,37 +145,44 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co
 		return err
 	}
 
-	var needsBuild bool
+	tool := types.Tool{
+		Source: types.ToolSource{
+			Repo: &types.Repo{
+				Root: runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"),
+			},
+		},
+	}
+	tag, err := golang.GetLatestTag(tool)
+	if err != nil {
+		return err
+	}
 
+	var needsDownloaded bool
 	// Check the last revision shasum and see if it is different from the current one.
 	lastRevision, err := os.ReadFile(m.credHelperDirs.RevisionFile)
-	if (err == nil && strings.TrimSpace(string(lastRevision)) != repo.Revision) || errors.Is(err, fs.ErrNotExist) {
+	if (err == nil && strings.TrimSpace(string(lastRevision)) != tool.Source.Repo.Root+tag) || errors.Is(err, fs.ErrNotExist) {
 		// Need to pull the latest version.
-		needsBuild = true
-		if err := git.Checkout(ctx, m.gitDir, repo.Root, repo.Revision, filepath.Join(m.credHelperDirs.RepoDir, repo.Revision)); err != nil {
-			return err
-		}
+		needsDownloaded = true
 		// Update the revision file to the new revision.
-		if err := os.WriteFile(m.credHelperDirs.RevisionFile, []byte(repo.Revision), 0644); err != nil {
+		if err = os.WriteFile(m.credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil {
 			return err
 		}
 	} else if err != nil {
 		return err
 	}
 
-	if !needsBuild {
-		// Check for the existence of the gptscript-credential-osxkeychain binary.
-		// If it's there, we have no need to build it and can just return.
-		if _, err := os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil {
+	if !needsDownloaded {
+		// Check for the existence of the credential helper binary.
+		// If it's there, we have no need to download it and can just return.
+		if _, err = os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil {
 			return nil
 		}
 	}
 
 	// Find the Go runtime and use it to build the credential helper.
-	for _, runtime := range m.runtimes {
-		if strings.HasPrefix(runtime.ID(), "go") {
-			goRuntime := runtime.(*golang.Runtime)
-			return goRuntime.BuildCredentialHelper(ctx, helperName, m.credHelperDirs, m.runtimeDir, repo.Revision, env)
+	for _, rt := range m.runtimes {
+		if strings.HasPrefix(rt.ID(), "go") {
+			return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, m.credHelperDirs.BinDir)
 		}
 	}
 

Diff for: pkg/repos/runtimes/golang/golang.go

@@ -18,7 +18,6 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/gptscript-ai/gptscript/pkg/credentials"
 	"github.com/gptscript-ai/gptscript/pkg/debugcmd"
 	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
@@ -97,6 +96,14 @@ type tag struct {
 	} `json:"commit"`
 }
 
+func GetLatestTag(tool types.Tool) (string, error) {
+	r, ok := getLatestRelease(tool)
+	if !ok {
+		return "", fmt.Errorf("failed to get latest release for %s", tool.Name)
+	}
+	return r.label, nil
+}
+
 func getLatestRelease(tool types.Tool) (*release, bool) {
 	if tool.Source.Repo == nil || !strings.HasPrefix(tool.Source.Repo.Root, "https://github.com/") {
 		return nil, false
@@ -116,11 +123,14 @@ func getLatestRelease(tool types.Tool) (*release, bool) {
 	account, repo := parts[1], parts[2]
 
 	resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/tags", account, repo))
-	if err != nil || resp.StatusCode != http.StatusOK {
+	if err != nil {
 		// ignore error
 		return nil, false
 	}
 	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, false
+	}
 
 	var tags []tag
 	if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil {
@@ -137,11 +147,14 @@ func getLatestRelease(tool types.Tool) (*release, bool) {
 	}
 
 	resp, err = client.Get(fmt.Sprintf("https://github.com/%s/%s/releases/latest", account, repo))
-	if err != nil || resp.StatusCode != http.StatusFound {
+	if err != nil {
 		// ignore error
 		return nil, false
 	}
 	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusFound {
+		return nil, false
+	}
 
 	target := resp.Header.Get("Location")
 	if target == "" {
@@ -212,7 +225,7 @@ func downloadBin(ctx context.Context, checksum, src, url, bin string) error {
 	return nil
 }
 
-func getChecksum(ctx context.Context, rel *release) string {
+func getChecksum(ctx context.Context, rel *release, artifactName string) string {
 	resp, err := get(ctx, rel.checksumTxt())
 	if err != nil {
 		// ignore error
@@ -223,7 +236,7 @@ func getChecksum(ctx context.Context, rel *release) string {
 	scan := bufio.NewScanner(resp.Body)
 	for scan.Scan() {
 		fields := strings.Fields(scan.Text())
-		if len(fields) == 2 && (fields[1] == rel.srcBinName() || fields[1] == "*"+rel.srcBinName()) {
+		if len(fields) == 2 && (fields[1] == artifactName || fields[1] == "*"+artifactName) {
 			return fields[0]
 		}
 	}
@@ -241,7 +254,7 @@ func (r *Runtime) Binary(ctx context.Context, tool types.Tool, _, toolSource str
 		return false, nil, nil
 	}
 
-	checksum := getChecksum(ctx, rel)
+	checksum := getChecksum(ctx, rel, rel.srcBinName())
 	if checksum == "" {
 		return false, nil, nil
 	}
@@ -268,30 +281,28 @@ func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource
 	return newEnv, nil
 }
 
-func (r *Runtime) BuildCredentialHelper(ctx context.Context, helperName string, credHelperDirs credentials.CredentialHelperDirs, dataRoot, revision string, env []string) error {
+func (r *Runtime) DownloadCredentialHelper(ctx context.Context, tool types.Tool, helperName, distInfo, suffix string, binDir string) error {
 	if helperName == "file" {
 		return nil
 	}
 
-	var suffix string
-	if helperName == "wincred" {
-		suffix = ".exe"
+	rel, ok := getLatestRelease(tool)
+	if !ok {
+		return fmt.Errorf("failed to find %s release", r.ID())
+	}
+	binaryName := "gptscript-credential-" + helperName
+	checksum := getChecksum(ctx, rel, binaryName+distInfo+suffix)
+	if checksum == "" {
+		return fmt.Errorf("failed to find %s release checksum for os=%s arch=%s", r.ID(), runtime.GOOS, runtime.GOARCH)
 	}
 
-	binPath, err := r.getRuntime(ctx, dataRoot)
-	if err != nil {
-		return err
+	url, _ := strings.CutSuffix(rel.binURL(), rel.srcBinName())
+	url += binaryName + distInfo + suffix
+	if err := downloadBin(ctx, checksum, strings.TrimSuffix(binDir, "bin"), url, binaryName+suffix); err != nil {
+		return fmt.Errorf("failed to download %s release for os=%s arch=%s: %w", r.ID(), runtime.GOOS, runtime.GOARCH, err)
 	}
-	newEnv := runtimeEnv.AppendPath(env, binPath)
 
-	log.InfofCtx(ctx, "Building credential helper %s", helperName)
-	cmd := debugcmd.New(ctx, filepath.Join(binPath, "go"),
-		"build", "-buildvcs=false", "-o",
-		filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix),
-		fmt.Sprintf("./%s/cmd/", helperName))
-	cmd.Env = stripGo(append(env, newEnv...))
-	cmd.Dir = filepath.Join(credHelperDirs.RepoDir, revision)
-	return cmd.Run()
+	return nil
 }
 
 func (r *Runtime) getReleaseAndDigest() (string, string, error) {

Diff for: pkg/runner/runtimemanager.go

@@ -45,6 +45,6 @@ func (r runtimeManagerLogger) EnsureCredentialHelpers(ctx context.Context) error
 	return r.rm.EnsureCredentialHelpers(mvl.WithInfo(ctx, r))
 }
 
-func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig, _ []string) error {
+func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig) error {
 	panic("not implemented")
 }