[runx] Proof of concept for runx integration (#1524)

## Summary

Inspired by golangci-lint being broken in nixpkgs, I wanted to see if I
could get a quick and dirty implementation so we can replace
golangci-lint from nixpkgs with runx. The current PR implements this
using `runx:<path>` syntax. Since we already use `github:<path>` for nix
flakes, I can't use that without breaking backwards compatibility. We
could overload it.

What this PR implements:

* Add and rm of runx, with and without versions.
* runx installed packages are added to PATH for run, shell, etc

Things that are not implemented by this PR:

* Search and validation. Currently trying to install a non-existing
package will fail with a semi-cryptic error.
* lockfile support. 

## How was it tested?

<img width="563" alt="image"
src="https://github.com/jetpack-io/devbox/assets/544948/1c0a5521-a6f9-420f-9178-4fc23a1bc059">

Author: mikeland73

.github/workflows/cli-tests.yaml

@@ -68,10 +68,11 @@ jobs:
           go-version-file: ./go.mod
           cache: false
 
-      - name: Install devbox
-        uses: jetpack-io/[email protected]
-        with:
-          enable-cache: true
+      # This can be reanabled once released version supports runx
+      # - name: Install devbox
+      #   uses: jetpack-io/[email protected]
+      #   with:
+      #     enable-cache: true
 
       - name: Mount golang cache
         uses: actions/cache@v3
@@ -82,7 +83,8 @@ jobs:
             ~/go/pkg
           key: go-${{ runner.os }}-${{ hashFiles('go.sum') }}
 
-      - run:  devbox run fmt
+      # Use main devbox for now to ensure it supports runx
+      - run:  go run ./cmd/devbox run fmt
 
       - name: golangci-lint
         uses: golangci/[email protected]

devbox.json

@@ -1,7 +1,7 @@
 {
   "packages": [
-    "[email protected]",
-    "go@latest"
+    "go@latest",
+    "runx:golangci/golangci-lint@latest"
   ],
   "env": {
     "GOENV": "off",
@@ -20,8 +20,8 @@
         "GOOS=linux GOARCH=arm64 go build -o dist/devbox-linux-arm64 ./cmd/devbox"
       ],
       "code": "code .",
-      "lint": "golangci-lint run",
       "fmt": "scripts/gofumpt.sh",
+      "lint": "golangci-lint run",
       "test": "go test -race -cover ./...",
       "update-examples": "devbox run build && go run testscripts/testrunner/updater/main.go"
     }

devbox.lock

@@ -21,24 +21,8 @@
         }
       }
     },
-    "[email protected]": {
-      "last_modified": "2023-05-01T16:53:22Z",
-      "resolved": "github:NixOS/nixpkgs/8670e496ffd093b60e74e7fa53526aa5920d09eb#golangci-lint",
-      "version": "1.52.2",
-      "systems": {
-        "aarch64-darwin": {
-          "store_path": "/nix/store/x2l9ljvl5fzffc7vggjk5yksrka9yra4-golangci-lint-1.52.2"
-        },
-        "aarch64-linux": {
-          "store_path": "/nix/store/k905aab7fj7dvf2hhsbb3mmizk052ad7-golangci-lint-1.52.2"
-        },
-        "x86_64-darwin": {
-          "store_path": "/nix/store/imxj7162whh9d56zpk1lzs1b3iw6wzp3-golangci-lint-1.52.2"
-        },
-        "x86_64-linux": {
-          "store_path": "/nix/store/nfhrynpdjd5yamz5snv2c377v1j9jdmx-golangci-lint-1.52.2"
-        }
-      }
+    "runx:golangci/golangci-lint@latest": {
+      "resolved": "runx:golangci/golangci-lint@latest"
     }
   }
 }

internal/devpkg/package.go

@@ -19,6 +19,7 @@ import (
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/devconfig"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/plugins"
@@ -93,6 +94,9 @@ func newPackage(raw string, isInstallable bool, locker lock.Locker) *Package {
 			normalizedURL += "#" + pkgURL.Fragment
 		}
 		pkgURL, _ = url.Parse(normalizedURL)
+	} else if pkgURL.Scheme == pkgtype.RunXScheme {
+		// THIS IS A HACK. These are not URLs and should not be treated as such
+		pkgURL.Path = pkgURL.Opaque
 	}
 
 	return &Package{URL: *pkgURL, lockfile: locker, Raw: raw, isInstallable: isInstallable}
@@ -107,11 +111,13 @@ func (p *Package) isLocal() bool {
 }
 
 // IsDevboxPackage specifies whether this package is a devbox package. Devbox
-// packages have the format `canonicalName@version`and can be resolved by devbox
-// search. This also returns true for legacy packages which are just an
-// attribute path. An explicit flake reference is _not_ a devbox package.
+// packages have the format `canonicalName@version`and can be resolved by
+// lockfile.Resolve (including runx packages)
+// This also returns true for legacy packages which are just
+// an attribute path. An explicit flake reference is _not_ a devbox package.
+// TODO: Consider renaming to IsResolvable
 func (p *Package) IsDevboxPackage() bool {
-	return p.Scheme == ""
+	return p.Scheme == "" || p.IsRunX()
 }
 
 // isGithub specifies whether this Package is referenced by a remote flake
@@ -401,7 +407,7 @@ func (p *Package) CanonicalName() string {
 	if !p.IsDevboxPackage() {
 		return ""
 	}
-	name, _, _ := strings.Cut(p.Path, "@")
+	name, _, _ := strings.Cut(p.Raw, "@")
 	return name
 }
 
@@ -519,3 +525,23 @@ func (p *Package) EnsureUninstallableIsInLockfile() error {
 	_, err := p.lockfile.Resolve(p.Raw)
 	return err
 }
+
+func (p *Package) IsRunX() bool {
+	return pkgtype.IsRunX(p.Raw)
+}
+
+func (p *Package) IsNix() bool {
+	return IsNix(p, 0)
+}
+
+func (p *Package) RunXPath() string {
+	return strings.TrimPrefix(p.Raw, pkgtype.RunXPrefix)
+}
+
+func IsNix(p *Package, _ int) bool {
+	return !p.IsRunX()
+}
+
+func IsRunX(p *Package, _ int) bool {
+	return p.IsRunX()
+}

internal/devpkg/package_test.go

@@ -226,3 +226,29 @@ func TestStorePathParts(t *testing.T) {
 		})
 	}
 }
+
+func TestCanonicalName(t *testing.T) {
+	tests := []struct {
+		pkgName      string
+		expectedName string
+	}{
+		{"go", "go"},
+		{"go@latest", "go"},
+		{"[email protected]", "go"},
+		{"runx:golangci/golangci-lint@latest", "runx:golangci/golangci-lint"},
+		{"runx:golangci/[email protected]", "runx:golangci/golangci-lint"},
+		{"runx:golangci/golangci-lint", "runx:golangci/golangci-lint"},
+		{"github:NixOS/nixpkgs/12345", ""},
+		{"path:/to/my/file", ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.pkgName, func(t *testing.T) {
+			pkg := PackageFromString(tt.pkgName, nil)
+			got := pkg.CanonicalName()
+			if got != tt.expectedName {
+				t.Errorf("Expected canonical name %q, but got %q", tt.expectedName, got)
+			}
+		})
+	}
+}

internal/devpkg/pkgtype/runx.go

@@ -0,0 +1,12 @@
+package pkgtype
+
+import "strings"
+
+const (
+	RunXScheme = "runx"
+	RunXPrefix = RunXScheme + ":"
+)
+
+func IsRunX(s string) bool {
+	return strings.HasPrefix(s, RunXPrefix)
+}

internal/devpkg/validation.go

@@ -8,6 +8,10 @@ import (
 )
 
 func (p *Package) ValidateExists() (bool, error) {
+	if p.IsRunX() {
+		// TODO implement runx validation
+		return true, nil
+	}
 	if p.isVersioned() && p.version() == "" {
 		return false, usererr.New("No version specified for %q.", p.Path)
 	}

internal/impl/devbox.go

@@ -27,6 +27,7 @@ import (
 	"go.jetpack.io/devbox/internal/searcher"
 	"go.jetpack.io/devbox/internal/shellgen"
 	"go.jetpack.io/devbox/internal/telemetry"
+	"go.jetpack.io/pkg/sandbox/runx"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cmdutil"
@@ -747,6 +748,7 @@ func (d *Devbox) StartProcessManager(
 // Note that the shellrc.tmpl template (which sources this environment) does
 // some additional processing. The computeNixEnv environment won't necessarily
 // represent the final "devbox run" or "devbox shell" environments.
+// TODO: Rename to computeDevboxEnv?
 func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (map[string]string, error) {
 	defer trace.StartRegion(ctx, "computeNixEnv").End()
 
@@ -874,6 +876,12 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 	})
 	debug.Log("PATH after filtering with buildInputs (%v) is: %s", buildInputs, nixEnvPath)
 
+	runXPaths, err := d.RunXPaths()
+	if err != nil {
+		return nil, err
+	}
+	nixEnvPath = envpath.JoinPathLists(nixEnvPath, runXPaths)
+
 	pathStack := envpath.Stack(env, originalEnv)
 	pathStack.Push(env, d.projectDirHash(), nixEnvPath, d.preservePathStack)
 	env["PATH"] = pathStack.Path(env)
@@ -978,6 +986,9 @@ func (d *Devbox) HasDeprecatedPackages() bool {
 }
 
 func (d *Devbox) findPackageByName(name string) (*devpkg.Package, error) {
+	if name == "" {
+		return nil, errors.New("package name cannot be empty")
+	}
 	results := map[*devpkg.Package]bool{}
 	for _, pkg := range d.configPackages() {
 		if pkg.Raw == name || pkg.CanonicalName() == name {
@@ -1194,3 +1205,16 @@ func (d *Devbox) PluginManager() *plugin.Manager {
 func (d *Devbox) Lockfile() *lock.File {
 	return d.lockfile
 }
+
+func (d *Devbox) RunXPaths() (string, error) {
+	packages := lo.Filter(d.InstallablePackages(), devpkg.IsRunX)
+	paths := []string{}
+	for _, pkg := range packages {
+		p, err := runx.Install(pkg.RunXPath())
+		if err != nil {
+			return "", err
+		}
+		paths = append(paths, p...)
+	}
+	return envpath.JoinPathLists(paths...), nil
+}

internal/impl/packages.go

@@ -18,6 +18,7 @@ import (
 	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/nix/nixprofile"
 	"go.jetpack.io/devbox/internal/shellgen"
+	"go.jetpack.io/pkg/sandbox/runx"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
@@ -66,6 +67,7 @@ func (d *Devbox) Add(ctx context.Context, platforms, excludePlatforms []string,
 		// match.
 		found, _ := d.findPackageByName(pkg.CanonicalName())
 		if found != nil {
+			ux.Finfo(d.stderr, "Replacing package %q in devbox.json\n", found.Raw)
 			if err := d.Remove(ctx, found.Raw); err != nil {
 				return err
 			}
@@ -234,6 +236,10 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod
 		return err
 	}
 
+	if err := d.InstallRunXPackages(); err != nil {
+		return err
+	}
+
 	if err := shellgen.GenerateForPrintEnv(ctx, d); err != nil {
 		return err
 	}
@@ -307,6 +313,9 @@ func (d *Devbox) syncPackagesToProfile(ctx context.Context, mode installMode) er
 		return err
 	}
 
+	// Remove non-nix packages from the list
+	packages = lo.Filter(packages, devpkg.IsNix)
+
 	if err := devpkg.FillNarInfoCache(ctx, packages...); err != nil {
 		return err
 	}
@@ -407,7 +416,7 @@ func (d *Devbox) addPackagesToProfile(ctx context.Context, pkgs []*devpkg.Packag
 
 	profileDir, err := d.profilePath()
 	if err != nil {
-		return err
+		return fmt.Errorf("error getting profile path: %w", err)
 	}
 
 	total := len(pkgs)
@@ -416,14 +425,14 @@ func (d *Devbox) addPackagesToProfile(ctx context.Context, pkgs []*devpkg.Packag
 
 		stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, pkg)
 
-		if err := nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{
+		if err = nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{
 			CustomStepMessage: stepMsg,
 			Lockfile:          d.lockfile,
 			Package:           pkg.Raw,
 			ProfilePath:       profileDir,
 			Writer:            d.stderr,
 		}); err != nil {
-			return err
+			return fmt.Errorf("error installing package %s: %w", pkg, err)
 		}
 	}
 
@@ -463,3 +472,20 @@ func resetProfileDirForFlakes(profileDir string) (err error) {
 
 	return errors.WithStack(os.Remove(profileDir))
 }
+
+func (d *Devbox) InstallRunXPackages() error {
+	for _, pkg := range d.InstallablePackages() {
+		if pkg.IsRunX() {
+			// TODO: Once resolve is implemented, we use whatever version is in the lockfile.
+			if _, err := d.lockfile.Resolve(pkg.Raw); err != nil {
+				return err
+			}
+			_, err := runx.Install(pkg.RunXPath())
+			if err != nil {
+				return fmt.Errorf("error installing runx package %s: %w", pkg, err)
+			}
+
+		}
+	}
+	return nil
+}

internal/lock/lockfile.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/searcher"
 
 	"go.jetpack.io/devbox/internal/cuecfg"
@@ -69,7 +70,12 @@ func (f *File) Resolve(pkg string) (*Package, error) {
 	if !hasEntry || entry.Resolved == "" {
 		locked := &Package{}
 		var err error
-		if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned {
+		if pkgtype.IsRunX(pkg) {
+			// TODO implement runx resolution. This can be done by reading the releases.json file
+			locked = &Package{
+				Resolved: pkg,
+			}
+		} else if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned {
 			locked, err = f.FetchResolvedPackage(pkg)
 			if err != nil {
 				return nil, err

internal/nix/search.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/pkg/errors"
-	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
 )
 
@@ -30,6 +30,10 @@ func (i *Info) String() string {
 }
 
 func Search(url string) (map[string]*Info, error) {
+	if strings.HasPrefix(url, "runx:") {
+		// TODO implement runx search
+		return map[string]*Info{}, nil
+	}
 	return searchSystem(url, "")
 }
 
@@ -98,7 +102,7 @@ func searchSystem(url, system string) (map[string]*Info, error) {
 	out, err := cmd.Output()
 	if err != nil {
 		// for now, assume all errors are invalid packages.
-		return nil, usererr.NewExecError(err)
+		return nil, fmt.Errorf("error searching for pkg %s: %w", url, err)
 	}
 	return parseSearchResults(out), nil
 }