Skip to content

Commit 43b8255

Browse files
johnrengelmanJohn Engelman
johnrengelman
authored and
John Engelman
committed
feat: Add support for tools from github enterprise.
1 parent 4fd8e8a commit 43b8255

File tree

4 files changed

+178
-75
lines changed

4 files changed

+178
-75
lines changed

Diff for: ‎docs/docs/04-command-line-reference/gptscript.md

+33-32
Original file line numberDiff line numberDiff line change
@@ -12,38 +12,39 @@ gptscript [flags] PROGRAM_FILE [INPUT...]
1212
### Options
1313

1414
```
15-
--cache-dir string Directory to store cache (default: $XDG_CACHE_HOME/gptscript) ($GPTSCRIPT_CACHE_DIR)
16-
--chat-state string The chat state to continue, or null to start a new chat and return the state ($GPTSCRIPT_CHAT_STATE)
17-
-C, --chdir string Change current working directory ($GPTSCRIPT_CHDIR)
18-
--color Use color in output (default true) ($GPTSCRIPT_COLOR)
19-
--config string Path to GPTScript config file ($GPTSCRIPT_CONFIG)
20-
--confirm Prompt before running potentially dangerous commands ($GPTSCRIPT_CONFIRM)
21-
--credential-context string Context name in which to store credentials ($GPTSCRIPT_CREDENTIAL_CONTEXT) (default "default")
22-
--credential-override strings Credentials to override (ex: --credential-override github.com/example/cred-tool:API_TOKEN=1234) ($GPTSCRIPT_CREDENTIAL_OVERRIDE)
23-
--debug Enable debug logging ($GPTSCRIPT_DEBUG)
24-
--debug-messages Enable logging of chat completion calls ($GPTSCRIPT_DEBUG_MESSAGES)
25-
--default-model string Default LLM model to use ($GPTSCRIPT_DEFAULT_MODEL) (default "gpt-4o")
26-
--default-model-provider string Default LLM model provider to use, this will override OpenAI settings ($GPTSCRIPT_DEFAULT_MODEL_PROVIDER)
27-
--disable-cache Disable caching of LLM API responses ($GPTSCRIPT_DISABLE_CACHE)
28-
--disable-tui Don't use chat TUI but instead verbose output ($GPTSCRIPT_DISABLE_TUI)
29-
--dump-state string Dump the internal execution state to a file ($GPTSCRIPT_DUMP_STATE)
30-
--events-stream-to string Stream events to this location, could be a file descriptor/handle (e.g. fd://2), filename, or named pipe (e.g. \\.\pipe\my-pipe) ($GPTSCRIPT_EVENTS_STREAM_TO)
31-
--force-chat Force an interactive chat session if even the top level tool is not a chat tool ($GPTSCRIPT_FORCE_CHAT)
32-
--force-sequential Force parallel calls to run sequentially ($GPTSCRIPT_FORCE_SEQUENTIAL)
33-
-h, --help help for gptscript
34-
-f, --input string Read input from a file ("-" for stdin) ($GPTSCRIPT_INPUT_FILE)
35-
--list-models List the models available and exit ($GPTSCRIPT_LIST_MODELS)
36-
--list-tools List built-in tools and exit ($GPTSCRIPT_LIST_TOOLS)
37-
--no-trunc Do not truncate long log messages ($GPTSCRIPT_NO_TRUNC)
38-
--openai-api-key string OpenAI API KEY ($OPENAI_API_KEY)
39-
--openai-base-url string OpenAI base URL ($OPENAI_BASE_URL)
40-
--openai-org-id string OpenAI organization ID ($OPENAI_ORG_ID)
41-
-o, --output string Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT)
42-
-q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
43-
--save-chat-state-file string A file to save the chat state to so that a conversation can be resumed with --chat-state ($GPTSCRIPT_SAVE_CHAT_STATE_FILE)
44-
--sub-tool string Use tool of this name, not the first tool in file ($GPTSCRIPT_SUB_TOOL)
45-
--ui Launch the UI ($GPTSCRIPT_UI)
46-
--workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
15+
--cache-dir string Directory to store cache (default: $XDG_CACHE_HOME/gptscript) ($GPTSCRIPT_CACHE_DIR)
16+
--chat-state string The chat state to continue, or null to start a new chat and return the state ($GPTSCRIPT_CHAT_STATE)
17+
-C, --chdir string Change current working directory ($GPTSCRIPT_CHDIR)
18+
--color Use color in output (default true) ($GPTSCRIPT_COLOR)
19+
--config string Path to GPTScript config file ($GPTSCRIPT_CONFIG)
20+
--confirm Prompt before running potentially dangerous commands ($GPTSCRIPT_CONFIRM)
21+
--credential-context string Context name in which to store credentials ($GPTSCRIPT_CREDENTIAL_CONTEXT) (default "default")
22+
--credential-override strings Credentials to override (ex: --credential-override github.com/example/cred-tool:API_TOKEN=1234) ($GPTSCRIPT_CREDENTIAL_OVERRIDE)
23+
--debug Enable debug logging ($GPTSCRIPT_DEBUG)
24+
--debug-messages Enable logging of chat completion calls ($GPTSCRIPT_DEBUG_MESSAGES)
25+
--default-model string Default LLM model to use ($GPTSCRIPT_DEFAULT_MODEL) (default "gpt-4o")
26+
--default-model-provider string Default LLM model provider to use, this will override OpenAI settings ($GPTSCRIPT_DEFAULT_MODEL_PROVIDER)
27+
--disable-cache Disable caching of LLM API responses ($GPTSCRIPT_DISABLE_CACHE)
28+
--disable-tui Don't use chat TUI but instead verbose output ($GPTSCRIPT_DISABLE_TUI)
29+
--dump-state string Dump the internal execution state to a file ($GPTSCRIPT_DUMP_STATE)
30+
--events-stream-to string Stream events to this location, could be a file descriptor/handle (e.g. fd://2), filename, or named pipe (e.g. \\.\pipe\my-pipe) ($GPTSCRIPT_EVENTS_STREAM_TO)
31+
--force-chat Force an interactive chat session if even the top level tool is not a chat tool ($GPTSCRIPT_FORCE_CHAT)
32+
--force-sequential Force parallel calls to run sequentially ($GPTSCRIPT_FORCE_SEQUENTIAL)
33+
--github-enterprise-hostname string The host name for a Github Enterprise instance to enable for remote loading ($GPTSCRIPT_GITHUB_ENTERPRISE_HOSTNAME)
34+
-h, --help help for gptscript
35+
-f, --input string Read input from a file ("-" for stdin) ($GPTSCRIPT_INPUT_FILE)
36+
--list-models List the models available and exit ($GPTSCRIPT_LIST_MODELS)
37+
--list-tools List built-in tools and exit ($GPTSCRIPT_LIST_TOOLS)
38+
--no-trunc Do not truncate long log messages ($GPTSCRIPT_NO_TRUNC)
39+
--openai-api-key string OpenAI API KEY ($OPENAI_API_KEY)
40+
--openai-base-url string OpenAI base URL ($OPENAI_BASE_URL)
41+
--openai-org-id string OpenAI organization ID ($OPENAI_ORG_ID)
42+
-o, --output string Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT)
43+
-q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
44+
--save-chat-state-file string A file to save the chat state to so that a conversation can be resumed with --chat-state ($GPTSCRIPT_SAVE_CHAT_STATE_FILE)
45+
--sub-tool string Use tool of this name, not the first tool in file ($GPTSCRIPT_SUB_TOOL)
46+
--ui Launch the UI ($GPTSCRIPT_UI)
47+
--workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
4748
```
4849

4950
### SEE ALSO

Diff for: ‎pkg/cli/gptscript.go

+25-19
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/gptscript-ai/gptscript/pkg/gptscript"
2424
"github.com/gptscript-ai/gptscript/pkg/input"
2525
"github.com/gptscript-ai/gptscript/pkg/loader"
26+
"github.com/gptscript-ai/gptscript/pkg/loader/github"
2627
"github.com/gptscript-ai/gptscript/pkg/monitor"
2728
"github.com/gptscript-ai/gptscript/pkg/mvl"
2829
"github.com/gptscript-ai/gptscript/pkg/openai"
@@ -54,25 +55,26 @@ type GPTScript struct {
5455
Output string `usage:"Save output to a file, or - for stdout" short:"o"`
5556
EventsStreamTo string `usage:"Stream events to this location, could be a file descriptor/handle (e.g. fd://2), filename, or named pipe (e.g. \\\\.\\pipe\\my-pipe)" name:"events-stream-to"`
5657
// Input should not be using GPTSCRIPT_INPUT env var because that is the same value that is set in tool executions
57-
Input string `usage:"Read input from a file (\"-\" for stdin)" short:"f" env:"GPTSCRIPT_INPUT_FILE"`
58-
SubTool string `usage:"Use tool of this name, not the first tool in file" local:"true"`
59-
Assemble bool `usage:"Assemble tool to a single artifact, saved to --output" hidden:"true" local:"true"`
60-
ListModels bool `usage:"List the models available and exit" local:"true"`
61-
ListTools bool `usage:"List built-in tools and exit" local:"true"`
62-
ListenAddress string `usage:"Server listen address" default:"127.0.0.1:0" hidden:"true"`
63-
Chdir string `usage:"Change current working directory" short:"C"`
64-
Daemon bool `usage:"Run tool as a daemon" local:"true" hidden:"true"`
65-
Ports string `usage:"The port range to use for ephemeral daemon ports (ex: 11000-12000)" hidden:"true"`
66-
CredentialContext string `usage:"Context name in which to store credentials" default:"default"`
67-
CredentialOverride []string `usage:"Credentials to override (ex: --credential-override github.com/example/cred-tool:API_TOKEN=1234)"`
68-
ChatState string `usage:"The chat state to continue, or null to start a new chat and return the state" local:"true"`
69-
ForceChat bool `usage:"Force an interactive chat session if even the top level tool is not a chat tool" local:"true"`
70-
ForceSequential bool `usage:"Force parallel calls to run sequentially" local:"true"`
71-
Workspace string `usage:"Directory to use for the workspace, if specified it will not be deleted on exit"`
72-
UI bool `usage:"Launch the UI" local:"true" name:"ui"`
73-
DisableTUI bool `usage:"Don't use chat TUI but instead verbose output" local:"true" name:"disable-tui"`
74-
SaveChatStateFile string `usage:"A file to save the chat state to so that a conversation can be resumed with --chat-state" local:"true"`
75-
DefaultModelProvider string `usage:"Default LLM model provider to use, this will override OpenAI settings"`
58+
Input string `usage:"Read input from a file (\"-\" for stdin)" short:"f" env:"GPTSCRIPT_INPUT_FILE"`
59+
SubTool string `usage:"Use tool of this name, not the first tool in file" local:"true"`
60+
Assemble bool `usage:"Assemble tool to a single artifact, saved to --output" hidden:"true" local:"true"`
61+
ListModels bool `usage:"List the models available and exit" local:"true"`
62+
ListTools bool `usage:"List built-in tools and exit" local:"true"`
63+
ListenAddress string `usage:"Server listen address" default:"127.0.0.1:0" hidden:"true"`
64+
Chdir string `usage:"Change current working directory" short:"C"`
65+
Daemon bool `usage:"Run tool as a daemon" local:"true" hidden:"true"`
66+
Ports string `usage:"The port range to use for ephemeral daemon ports (ex: 11000-12000)" hidden:"true"`
67+
CredentialContext string `usage:"Context name in which to store credentials" default:"default"`
68+
CredentialOverride []string `usage:"Credentials to override (ex: --credential-override github.com/example/cred-tool:API_TOKEN=1234)"`
69+
ChatState string `usage:"The chat state to continue, or null to start a new chat and return the state" local:"true"`
70+
ForceChat bool `usage:"Force an interactive chat session if even the top level tool is not a chat tool" local:"true"`
71+
ForceSequential bool `usage:"Force parallel calls to run sequentially" local:"true"`
72+
Workspace string `usage:"Directory to use for the workspace, if specified it will not be deleted on exit"`
73+
UI bool `usage:"Launch the UI" local:"true" name:"ui"`
74+
DisableTUI bool `usage:"Don't use chat TUI but instead verbose output" local:"true" name:"disable-tui"`
75+
SaveChatStateFile string `usage:"A file to save the chat state to so that a conversation can be resumed with --chat-state" local:"true"`
76+
DefaultModelProvider string `usage:"Default LLM model provider to use, this will override OpenAI settings"`
77+
GithubEnterpriseHostname string `usage:"The host name for a Github Enterprise instance to enable for remote loading" local:"true"`
7678

7779
readData []byte
7880
}
@@ -334,6 +336,10 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) (retErr error) {
334336
return err
335337
}
336338

339+
if r.GithubEnterpriseHostname != "" {
340+
loader.AddVSC(github.LoaderForPrefix(r.GithubEnterpriseHostname))
341+
}
342+
337343
// If the user is trying to launch the chat-builder UI, then set up the tool and options here.
338344
if r.UI {
339345
if os.Getenv(system.BinEnvVar) == "" {

Diff for: ‎pkg/loader/github/github.go

+63-24
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package github
22

33
import (
44
"context"
5+
"crypto/tls"
56
"encoding/json"
67
"fmt"
78
"io"
@@ -18,52 +19,63 @@ import (
1819
"github.com/gptscript-ai/gptscript/pkg/types"
1920
)
2021

21-
const (
22-
GithubPrefix = "github.com/"
23-
githubRepoURL = "https://github.com/%s/%s.git"
24-
githubDownloadURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
25-
githubCommitURL = "https://api.github.com/repos/%s/%s/commits/%s"
26-
)
22+
type GithubConfig struct {
23+
Prefix string
24+
RepoURL string
25+
DownloadURL string
26+
CommitURL string
27+
AuthToken string
28+
}
2729

2830
var (
29-
githubAuthToken = os.Getenv("GITHUB_AUTH_TOKEN")
30-
log = mvl.Package()
31+
log = mvl.Package()
32+
defaultGithubConfig = &GithubConfig{
33+
Prefix: "github.com/",
34+
RepoURL: "https://github.com/%s/%s.git",
35+
DownloadURL: "https://raw.githubusercontent.com/%s/%s/%s/%s",
36+
CommitURL: "https://api.github.com/repos/%s/%s/commits/%s",
37+
AuthToken: os.Getenv("GITHUB_AUTH_TOKEN"),
38+
}
3139
)
3240

3341
func init() {
3442
loader.AddVSC(Load)
3543
}
3644

37-
func getCommitLsRemote(ctx context.Context, account, repo, ref string) (string, error) {
38-
url := fmt.Sprintf(githubRepoURL, account, repo)
45+
func getCommitLsRemote(ctx context.Context, account, repo, ref string, config *GithubConfig) (string, error) {
46+
url := fmt.Sprintf(config.RepoURL, account, repo)
3947
return git.LsRemote(ctx, url, ref)
4048
}
4149

4250
// regexp to match a git commit id
4351
var commitRegexp = regexp.MustCompile("^[a-f0-9]{40}$")
4452

45-
func getCommit(ctx context.Context, account, repo, ref string) (string, error) {
53+
func getCommit(ctx context.Context, account, repo, ref string, config *GithubConfig) (string, error) {
4654
if commitRegexp.MatchString(ref) {
4755
return ref, nil
4856
}
4957

50-
url := fmt.Sprintf(githubCommitURL, account, repo, ref)
58+
url := fmt.Sprintf(config.CommitURL, account, repo, ref)
5159
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
5260
if err != nil {
5361
return "", fmt.Errorf("failed to create request of %s/%s at %s: %w", account, repo, url, err)
5462
}
5563

56-
if githubAuthToken != "" {
57-
req.Header.Add("Authorization", "Bearer "+githubAuthToken)
64+
if config.AuthToken != "" {
65+
req.Header.Add("Authorization", "Bearer "+config.AuthToken)
5866
}
5967

60-
resp, err := http.DefaultClient.Do(req)
68+
client := http.DefaultClient
69+
if req.Host == config.Prefix && strings.ToLower(os.Getenv("GH_ENTERPRISE_SKIP_VERIFY")) == "true" {
70+
client = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
71+
}
72+
resp, err := client.Do(req)
6173
if err != nil {
6274
return "", err
6375
} else if resp.StatusCode != http.StatusOK {
6476
c, _ := io.ReadAll(resp.Body)
6577
resp.Body.Close()
66-
commit, fallBackErr := getCommitLsRemote(ctx, account, repo, ref)
78+
commit, fallBackErr := getCommitLsRemote(ctx, account, repo, ref, config)
6779
if fallBackErr == nil {
6880
return commit, nil
6981
}
@@ -88,8 +100,28 @@ func getCommit(ctx context.Context, account, repo, ref string) (string, error) {
88100
return commit.SHA, nil
89101
}
90102

91-
func Load(ctx context.Context, _ *cache.Client, urlName string) (string, string, *types.Repo, bool, error) {
92-
if !strings.HasPrefix(urlName, GithubPrefix) {
103+
func LoaderForPrefix(prefix string) func(context.Context, *cache.Client, string) (string, string, *types.Repo, bool, error) {
104+
return func(ctx context.Context, c *cache.Client, urlName string) (string, string, *types.Repo, bool, error) {
105+
return LoadWithConfig(ctx, c, urlName, NewGithubEnterpriseConfig(prefix))
106+
}
107+
}
108+
109+
func Load(ctx context.Context, c *cache.Client, urlName string) (string, string, *types.Repo, bool, error) {
110+
return LoadWithConfig(ctx, c, urlName, defaultGithubConfig)
111+
}
112+
113+
func NewGithubEnterpriseConfig(prefix string) *GithubConfig {
114+
return &GithubConfig{
115+
Prefix: prefix,
116+
RepoURL: fmt.Sprintf("https://%s/%%s/%%s.git", prefix),
117+
DownloadURL: fmt.Sprintf("https://raw.%s/%%s/%%s/%%s/%%s", prefix),
118+
CommitURL: fmt.Sprintf("https://%s/api/v3/repos/%%s/%%s/commits/%%s", prefix),
119+
AuthToken: os.Getenv("GH_ENTERPRISE_TOKEN"),
120+
}
121+
}
122+
123+
func LoadWithConfig(ctx context.Context, _ *cache.Client, urlName string, config *GithubConfig) (string, string, *types.Repo, bool, error) {
124+
if !strings.HasPrefix(urlName, config.Prefix) {
93125
return "", "", nil, false, nil
94126
}
95127

@@ -107,12 +139,12 @@ func Load(ctx context.Context, _ *cache.Client, urlName string) (string, string,
107139
account, repo := parts[1], parts[2]
108140
path := strings.Join(parts[3:], "/")
109141

110-
ref, err := getCommit(ctx, account, repo, ref)
142+
ref, err := getCommit(ctx, account, repo, ref, config)
111143
if err != nil {
112144
return "", "", nil, false, err
113145
}
114146

115-
downloadURL := fmt.Sprintf(githubDownloadURL, account, repo, ref, path)
147+
downloadURL := fmt.Sprintf(config.DownloadURL, account, repo, ref, path)
116148
if path == "" || path == "/" || !strings.Contains(parts[len(parts)-1], ".") {
117149
var (
118150
testPath string
@@ -124,13 +156,20 @@ func Load(ctx context.Context, _ *cache.Client, urlName string) (string, string,
124156
} else {
125157
testPath = path + "/" + ext
126158
}
127-
testURL = fmt.Sprintf(githubDownloadURL, account, repo, ref, testPath)
159+
testURL = fmt.Sprintf(config.DownloadURL, account, repo, ref, testPath)
128160
if i == len(types.DefaultFiles)-1 {
129161
// no reason to test the last one, we are just going to use it. Being that the default list is only
130162
// two elements this loop could have been one check, but hey over-engineered code ftw.
131163
break
132164
}
133-
if resp, err := http.Head(testURL); err == nil {
165+
headReq, err := http.NewRequest("HEAD", testURL, nil)
166+
if err != nil {
167+
break
168+
}
169+
if config.AuthToken != "" {
170+
headReq.Header.Add("Authorization", "Bearer "+config.AuthToken)
171+
}
172+
if resp, err := http.DefaultClient.Do(headReq); err == nil {
134173
_ = resp.Body.Close()
135174
if resp.StatusCode == 200 {
136175
break
@@ -141,9 +180,9 @@ func Load(ctx context.Context, _ *cache.Client, urlName string) (string, string,
141180
path = testPath
142181
}
143182

144-
return downloadURL, githubAuthToken, &types.Repo{
183+
return downloadURL, config.AuthToken, &types.Repo{
145184
VCS: "git",
146-
Root: fmt.Sprintf(githubRepoURL, account, repo),
185+
Root: fmt.Sprintf(config.RepoURL, account, repo),
147186
Path: gpath.Dir(path),
148187
Name: gpath.Base(path),
149188
Revision: ref,

0 commit comments

Comments
 (0)