diff --git a/pkg/cli/credential.go b/pkg/cli/credential.go index b0c4a30a..cb000125 100644 --- a/pkg/cli/credential.go +++ b/pkg/cli/credential.go @@ -58,7 +58,7 @@ func (c *Credential) Run(cmd *cobra.Command, _ []string) error { opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) } - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil { + if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } diff --git a/pkg/cli/credential_delete.go b/pkg/cli/credential_delete.go index 9c986c54..4e9919df 100644 --- a/pkg/cli/credential_delete.go +++ b/pkg/cli/credential_delete.go @@ -40,7 +40,7 @@ func (c *Delete) Run(cmd *cobra.Command, args []string) error { opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) } - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil { + if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } diff --git a/pkg/cli/credential_show.go b/pkg/cli/credential_show.go index ccfe3675..fac1b719 100644 --- a/pkg/cli/credential_show.go +++ b/pkg/cli/credential_show.go @@ -42,7 +42,7 @@ func (c *Show) Run(cmd *cobra.Command, args []string) error { opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) } - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil { + if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go index 367b4d1d..70f31e97 100644 --- a/pkg/credentials/util.go +++ b/pkg/credentials/util.go @@ -5,7 +5,7 @@ import ( ) type CredentialHelperDirs struct { - RevisionFile, LastCheckedFile, BinDir, RepoDir, HelperDir string + RevisionFile, LastCheckedFile, BinDir string } func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs { @@ -13,7 +13,5 @@ func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs { RevisionFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "revision"), LastCheckedFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "last-checked"), BinDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "bin"), - RepoDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "repo"), - HelperDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers"), } } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 20ca43a9..f8fd8154 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -22,7 +22,7 @@ type Model interface { type RuntimeManager interface { GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) EnsureCredentialHelpers(ctx context.Context) error - SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error + SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error } type Engine struct { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index 43f429fc..755fe632 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -99,7 +99,7 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) { opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir()) } - if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg, opts.Env); err != nil { + if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil { return nil, err } diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 416f4c61..8981d1fa 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "strings" "sync" "time" @@ -15,15 +16,13 @@ import ( "github.com/BurntSushi/locker" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" - "github.com/gptscript-ai/gptscript/pkg/loader/github" "github.com/gptscript-ai/gptscript/pkg/repos/git" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/types" ) -const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-helpers" - type Runtime interface { ID() string Supports(tool types.Tool, cmd []string) bool @@ -68,7 +67,6 @@ type credHelperConfig struct { lock sync.Mutex initialized bool cliCfg *config.CLIConfig - env []string } func New(cacheDir string, runtimes ...Runtime) *Manager { @@ -90,7 +88,7 @@ func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error { defer m.credHelperConfig.lock.Unlock() if !m.credHelperConfig.initialized { - if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg, m.credHelperConfig.env); err != nil { + if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg); err != nil { return err } m.credHelperConfig.initialized = true @@ -99,27 +97,28 @@ func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error { return nil } -func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig, env []string) error { +func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig) error { m.credHelperConfig = &credHelperConfig{ cliCfg: cliCfg, - env: env, } return nil } -func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error { +func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error { var ( - helperName = cliCfg.CredentialsStore - suffix string + helperName = cliCfg.CredentialsStore + distInfo, suffix string ) - if helperName == "wincred" { - suffix = ".exe" - } - - // The file helper is built-in and does not need to be compiled. + // The file helper is built-in and does not need to be downloaded. if helperName == "file" { return nil } + switch helperName { + case "wincred": + suffix = ".exe" + default: + distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH) + } locker.Lock("gptscript-credential-helpers") defer locker.Unlock("gptscript-credential-helpers") @@ -137,13 +136,7 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co } } - // Load the credential helpers repo information. - _, _, repo, _, err := github.Load(ctx, nil, credentialHelpersRepo) - if err != nil { - return err - } - - if err := os.MkdirAll(m.credHelperDirs.HelperDir, 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(m.credHelperDirs.LastCheckedFile), 0755); err != nil { return err } @@ -152,37 +145,44 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co return err } - var needsBuild bool + tool := types.Tool{ + Source: types.ToolSource{ + Repo: &types.Repo{ + Root: runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"), + }, + }, + } + tag, err := golang.GetLatestTag(tool) + if err != nil { + return err + } + var needsDownloaded bool // Check the last revision shasum and see if it is different from the current one. lastRevision, err := os.ReadFile(m.credHelperDirs.RevisionFile) - if (err == nil && strings.TrimSpace(string(lastRevision)) != repo.Revision) || errors.Is(err, fs.ErrNotExist) { + if (err == nil && strings.TrimSpace(string(lastRevision)) != tool.Source.Repo.Root+tag) || errors.Is(err, fs.ErrNotExist) { // Need to pull the latest version. - needsBuild = true - if err := git.Checkout(ctx, m.gitDir, repo.Root, repo.Revision, filepath.Join(m.credHelperDirs.RepoDir, repo.Revision)); err != nil { - return err - } + needsDownloaded = true // Update the revision file to the new revision. - if err := os.WriteFile(m.credHelperDirs.RevisionFile, []byte(repo.Revision), 0644); err != nil { + if err = os.WriteFile(m.credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil { return err } } else if err != nil { return err } - if !needsBuild { - // Check for the existence of the gptscript-credential-osxkeychain binary. - // If it's there, we have no need to build it and can just return. - if _, err := os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { + if !needsDownloaded { + // Check for the existence of the credential helper binary. + // If it's there, we have no need to download it and can just return. + if _, err = os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { return nil } } // Find the Go runtime and use it to build the credential helper. - for _, runtime := range m.runtimes { - if strings.HasPrefix(runtime.ID(), "go") { - goRuntime := runtime.(*golang.Runtime) - return goRuntime.BuildCredentialHelper(ctx, helperName, m.credHelperDirs, m.runtimeDir, repo.Revision, env) + for _, rt := range m.runtimes { + if strings.HasPrefix(rt.ID(), "go") { + return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, m.credHelperDirs.BinDir) } } diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 3514fadb..52e8fe0b 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -18,7 +18,6 @@ import ( "runtime" "strings" - "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/debugcmd" runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" @@ -97,6 +96,14 @@ type tag struct { } `json:"commit"` } +func GetLatestTag(tool types.Tool) (string, error) { + r, ok := getLatestRelease(tool) + if !ok { + return "", fmt.Errorf("failed to get latest release for %s", tool.Name) + } + return r.label, nil +} + func getLatestRelease(tool types.Tool) (*release, bool) { if tool.Source.Repo == nil || !strings.HasPrefix(tool.Source.Repo.Root, "https://github.com/") { return nil, false @@ -116,11 +123,14 @@ func getLatestRelease(tool types.Tool) (*release, bool) { account, repo := parts[1], parts[2] resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/tags", account, repo)) - if err != nil || resp.StatusCode != http.StatusOK { + if err != nil { // ignore error return nil, false } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, false + } var tags []tag if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil { @@ -137,11 +147,14 @@ func getLatestRelease(tool types.Tool) (*release, bool) { } resp, err = client.Get(fmt.Sprintf("https://github.com/%s/%s/releases/latest", account, repo)) - if err != nil || resp.StatusCode != http.StatusFound { + if err != nil { // ignore error return nil, false } defer resp.Body.Close() + if resp.StatusCode != http.StatusFound { + return nil, false + } target := resp.Header.Get("Location") if target == "" { @@ -212,7 +225,7 @@ func downloadBin(ctx context.Context, checksum, src, url, bin string) error { return nil } -func getChecksum(ctx context.Context, rel *release) string { +func getChecksum(ctx context.Context, rel *release, artifactName string) string { resp, err := get(ctx, rel.checksumTxt()) if err != nil { // ignore error @@ -223,7 +236,7 @@ func getChecksum(ctx context.Context, rel *release) string { scan := bufio.NewScanner(resp.Body) for scan.Scan() { fields := strings.Fields(scan.Text()) - if len(fields) == 2 && (fields[1] == rel.srcBinName() || fields[1] == "*"+rel.srcBinName()) { + if len(fields) == 2 && (fields[1] == artifactName || fields[1] == "*"+artifactName) { return fields[0] } } @@ -241,7 +254,7 @@ func (r *Runtime) Binary(ctx context.Context, tool types.Tool, _, toolSource str return false, nil, nil } - checksum := getChecksum(ctx, rel) + checksum := getChecksum(ctx, rel, rel.srcBinName()) if checksum == "" { return false, nil, nil } @@ -268,30 +281,28 @@ func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource return newEnv, nil } -func (r *Runtime) BuildCredentialHelper(ctx context.Context, helperName string, credHelperDirs credentials.CredentialHelperDirs, dataRoot, revision string, env []string) error { +func (r *Runtime) DownloadCredentialHelper(ctx context.Context, tool types.Tool, helperName, distInfo, suffix string, binDir string) error { if helperName == "file" { return nil } - var suffix string - if helperName == "wincred" { - suffix = ".exe" + rel, ok := getLatestRelease(tool) + if !ok { + return fmt.Errorf("failed to find %s release", r.ID()) + } + binaryName := "gptscript-credential-" + helperName + checksum := getChecksum(ctx, rel, binaryName+distInfo+suffix) + if checksum == "" { + return fmt.Errorf("failed to find %s release checksum for os=%s arch=%s", r.ID(), runtime.GOOS, runtime.GOARCH) } - binPath, err := r.getRuntime(ctx, dataRoot) - if err != nil { - return err + url, _ := strings.CutSuffix(rel.binURL(), rel.srcBinName()) + url += binaryName + distInfo + suffix + if err := downloadBin(ctx, checksum, strings.TrimSuffix(binDir, "bin"), url, binaryName+suffix); err != nil { + return fmt.Errorf("failed to download %s release for os=%s arch=%s: %w", r.ID(), runtime.GOOS, runtime.GOARCH, err) } - newEnv := runtimeEnv.AppendPath(env, binPath) - log.InfofCtx(ctx, "Building credential helper %s", helperName) - cmd := debugcmd.New(ctx, filepath.Join(binPath, "go"), - "build", "-buildvcs=false", "-o", - filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix), - fmt.Sprintf("./%s/cmd/", helperName)) - cmd.Env = stripGo(append(env, newEnv...)) - cmd.Dir = filepath.Join(credHelperDirs.RepoDir, revision) - return cmd.Run() + return nil } func (r *Runtime) getReleaseAndDigest() (string, string, error) { diff --git a/pkg/runner/runtimemanager.go b/pkg/runner/runtimemanager.go index e1c5a4c6..ed191d15 100644 --- a/pkg/runner/runtimemanager.go +++ b/pkg/runner/runtimemanager.go @@ -45,6 +45,6 @@ func (r runtimeManagerLogger) EnsureCredentialHelpers(ctx context.Context) error return r.rm.EnsureCredentialHelpers(mvl.WithInfo(ctx, r)) } -func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig, _ []string) error { +func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig) error { panic("not implemented") }