feat: add basic bash support for windows

Author: ibuildthecloud

pkg/engine/cmd.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -196,10 +197,9 @@ var ignoreENV = map[string]struct{}{
 }
 
 func appendEnv(envs []string, k, v string) []string {
-	for _, k := range []string{k, env.ToEnvLike(k)} {
-		if _, ignore := ignoreENV[k]; !ignore {
-			envs = append(envs, k+"="+v)
-		}
+	k = env.ToEnvLike(k)
+	if _, ignore := ignoreENV[k]; !ignore {
+		envs = append(envs, k+"="+v)
 	}
 	return envs
 }
@@ -262,6 +262,10 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
 		})
 	}
 
+	if runtime.GOOS == "windows" && (args[0] == "/bin/bash" || args[0] == "/bin/sh") {
+		args[0] = path.Base(args[0])
+	}
+
 	if runtime.GOOS == "windows" && (args[0] == "/usr/bin/env" || args[0] == "/bin/env") {
 		args = args[1:]
 	}

pkg/repos/download/extract.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/mholt/archiver/v4"
@@ -60,6 +62,18 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error {
 		return err
 	}
 
+	bin := path.Base(parsedURL.Path)
+	if strings.HasSuffix(bin, ".exe") {
+		dst, err := os.Create(filepath.Join(targetDir, bin))
+		if err != nil {
+			return err
+		}
+		defer dst.Close()
+
+		_, err = io.Copy(dst, tmpFile)
+		return err
+	}
+
 	format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), tmpFile)
 	if err != nil {
 		return err

pkg/repos/get.go

@@ -15,6 +15,7 @@ import (
 	"github.com/BurntSushi/locker"
 	"github.com/gptscript-ai/gptscript/pkg/config"
 	"github.com/gptscript-ai/gptscript/pkg/credentials"
+	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/loader/github"
 	"github.com/gptscript-ai/gptscript/pkg/repos/git"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang"
@@ -51,13 +52,22 @@ type Manager struct {
 	credHelperDirs   credentials.CredentialHelperDirs
 	runtimes         []Runtime
 	credHelperConfig *credHelperConfig
+	supportLocal     bool
 }
 
 type credHelperConfig struct {
 	lock        sync.Mutex
 	initialized bool
 	cliCfg      *config.CLIConfig
 	env         []string
+	storageDir  string
+	gitDir      string
+	runtimeDir  string
+	runtimes    []Runtime
+}
+
+func (m *Manager) SetSupportLocal() {
+	m.supportLocal = true
 }
 
 func New(cacheDir string, runtimes ...Runtime) *Manager {
@@ -200,8 +210,14 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 	_ = os.RemoveAll(doneFile)
 	_ = os.RemoveAll(target)
 
-	if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil {
-		return "", nil, err
+	if tool.Source.Repo.VCS == "git" {
+		if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil {
+			return "", nil, err
+		}
+	} else {
+		if err := os.MkdirAll(target, 0755); err != nil {
+			return "", nil, err
+		}
 	}
 
 	newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env)
@@ -227,12 +243,25 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 }
 
 func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) {
-	if tool.Source.Repo == nil {
-		return tool.WorkingDir, env, nil
-	}
+	var isLocal bool
+	if !m.supportLocal {
+		if tool.Source.Repo == nil {
+			return tool.WorkingDir, env, nil
+		}
 
-	if tool.Source.Repo.VCS != "git" {
-		return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID)
+		if tool.Source.Repo.VCS != "git" {
+			return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID)
+		}
+	} else if tool.Source.Repo == nil {
+		isLocal = true
+		id := hash.Digest(tool)[:12]
+		tool.Source.Repo = &types.Repo{
+			VCS:      "<local>",
+			Root:     id,
+			Path:     "/",
+			Name:     id,
+			Revision: id,
+		}
 	}
 
 	for _, runtime := range m.runtimes {
@@ -242,5 +271,9 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st
 		}
 	}
 
+	if isLocal {
+		return tool.WorkingDir, env, nil
+	}
+
 	return m.setup(ctx, &noopRuntime{}, tool, env)
 }

pkg/repos/runtimes/busybox/SHASUMS256.txt

@@ -0,0 +1 @@
+6d2dfd1c1412c3550a89071a1b36a6f6073844320e687343d1dfc72719ecb8d9  FRP-5301-gda71f7c57/busybox-w64-FRP-5301-gda71f7c57.exe

pkg/repos/runtimes/busybox/busybox.go

@@ -0,0 +1,107 @@
+package busybox
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
+	"github.com/gptscript-ai/gptscript/pkg/hash"
+	"github.com/gptscript-ai/gptscript/pkg/repos/download"
+)
+
+//go:embed SHASUMS256.txt
+var releasesData []byte
+
+const downloadURL = "https://github.com/gptscript-ai/busybox-w32/releases/download/%s"
+
+type Runtime struct {
+}
+
+func (r *Runtime) ID() string {
+	return "busybox"
+}
+
+func (r *Runtime) Supports(cmd []string) bool {
+	if runtime.GOOS != "windows" {
+		return false
+	}
+	for _, bin := range []string{"bash", "sh", "/bin/sh", "/bin/bash"} {
+		if runtimeEnv.Matches(cmd, bin) {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *Runtime) Setup(ctx context.Context, dataRoot, _ string, env []string) ([]string, error) {
+	binPath, err := r.getRuntime(ctx, dataRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	newEnv := runtimeEnv.AppendPath(env, binPath)
+	return newEnv, nil
+}
+
+func (r *Runtime) getReleaseAndDigest() (string, string, error) {
+	scanner := bufio.NewScanner(bytes.NewReader(releasesData))
+	for scanner.Scan() {
+		line := scanner.Text()
+		fields := strings.Fields(line)
+		return fmt.Sprintf(downloadURL, fields[1]), fields[0], nil
+	}
+
+	return "", "", fmt.Errorf("failed to find %s release", r.ID())
+}
+
+func (r *Runtime) getRuntime(ctx context.Context, cwd string) (string, error) {
+	url, sha, err := r.getReleaseAndDigest()
+	if err != nil {
+		return "", err
+	}
+
+	target := filepath.Join(cwd, "busybox", hash.ID(url, sha))
+	if _, err := os.Stat(target); err == nil {
+		return target, nil
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return "", err
+	}
+
+	log.Infof("Downloading Busybox")
+	tmp := target + ".download"
+	defer os.RemoveAll(tmp)
+
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		return "", err
+	}
+
+	if err := download.Extract(ctx, url, sha, tmp); err != nil {
+		return "", err
+	}
+
+	bbExe := filepath.Join(tmp, path.Base(url))
+
+	cmd := exec.Command(bbExe, "--install", ".")
+	cmd.Dir = filepath.Dir(bbExe)
+
+	if err := cmd.Run(); err != nil {
+		return "", err
+	}
+
+	if err := os.Rename(tmp, target); err != nil {
+		return "", err
+	}
+
+	return target, nil
+}

pkg/repos/runtimes/busybox/busybox_test.go

@@ -0,0 +1,41 @@
+package busybox
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/adrg/xdg"
+	"github.com/samber/lo"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testCacheHome = lo.Must(xdg.CacheFile("gptscript-test-cache/runtime"))
+)
+
+func firstPath(s []string) string {
+	_, p, _ := strings.Cut(s[0], "=")
+	return strings.Split(p, string(os.PathListSeparator))[0]
+}
+
+func TestRuntime(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skip()
+	}
+
+	r := Runtime{}
+
+	s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
+	require.NoError(t, err)
+	_, err = os.Stat(filepath.Join(firstPath(s), "busybox.exe"))
+	if errors.Is(err, fs.ErrNotExist) {
+		_, err = os.Stat(filepath.Join(firstPath(s), "busybox"))
+	}
+	require.NoError(t, err)
+}

pkg/repos/runtimes/busybox/log.go

@@ -0,0 +1,5 @@
+package busybox
+
+import "github.com/gptscript-ai/gptscript/pkg/mvl"
+
+var log = mvl.Package()

pkg/repos/runtimes/default.go

@@ -3,12 +3,14 @@ package runtimes
 import (
 	"github.com/gptscript-ai/gptscript/pkg/engine"
 	"github.com/gptscript-ai/gptscript/pkg/repos"
+	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/busybox"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/node"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/python"
 )
 
 var Runtimes = []repos.Runtime{
+	&busybox.Runtime{},
 	&python.Runtime{
 		Version: "3.12",
 		Default: true,

pkg/tests/runner_test.go

@@ -751,9 +751,6 @@ func TestGlobalErr(t *testing.T) {
 }
 
 func TestContextArg(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
 	runner := tester.NewRunner(t)
 	x, err := runner.Run("", `{
 "file": "foo.db"

pkg/tests/testdata/TestContextArg/other.gpt

@@ -2,5 +2,5 @@ name: fromcontext
 args: first: an arg
 args: second: an arg
 
-#!/bin/bash
-echo this is from other context ${first} and then ${second}
+#!/usr/bin/env bash
+echo this is from other context ${FIRST} and then ${SECOND}

pkg/tests/testdata/TestContextArg/test.gpt

@@ -9,4 +9,4 @@ name: fromcontext
 args: first: an arg
 
 #!/bin/bash
-echo this is from context -- ${first}
+echo this is from context -- ${FIRST}

pkg/tests/tester/runner.go

@@ -8,8 +8,11 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/adrg/xdg"
 	"github.com/gptscript-ai/gptscript/pkg/credentials"
 	"github.com/gptscript-ai/gptscript/pkg/loader"
+	"github.com/gptscript-ai/gptscript/pkg/repos"
+	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes"
 	"github.com/gptscript-ai/gptscript/pkg/runner"
 	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/hexops/autogold/v2"
@@ -171,8 +174,15 @@ func NewRunner(t *testing.T) *Runner {
 		t: t,
 	}
 
+	cacheDir, err := xdg.CacheFile("gptscript-test-cache/runtime")
+	require.NoError(t, err)
+
+	rm := runtimes.Default(cacheDir)
+	rm.(*repos.Manager).SetSupportLocal()
+
 	run, err := runner.New(c, credentials.NoopStore{}, runner.Options{
-		Sequential: true,
+		Sequential:     true,
+		RuntimeManager: rm,
 	})
 	require.NoError(t, err)