[runx] Add runx validate, lockfile resolve

.github/workflows/cli-tests.yaml

@@ -37,6 +37,7 @@ defaults:
 
 env:
   HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  DEVBOX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   HOMEBREW_NO_ANALYTICS: 1
   HOMEBREW_NO_AUTO_UPDATE: 1
   HOMEBREW_NO_EMOJI: 1

devbox.lock

@@ -22,7 +22,8 @@
       }
     },
     "runx:golangci/golangci-lint@latest": {
-      "resolved": "runx:golangci/golangci-lint@latest"
+      "resolved": "golangci/[email protected]",
+      "version": "v1.54.2"
     }
   }
 }

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/wk8/go-ordered-map/v2 v2.1.8
 	github.com/zealic/go2node v0.1.0
-	go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3
+	go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
 	golang.org/x/mod v0.12.0
 	golang.org/x/sync v0.3.0

go.sum

@@ -337,6 +337,8 @@ github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA=
 github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg=
 go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3 h1:aMydtVCHn7dfotOyV41VAxX5b5OOsCc4TxOXwDt38Yw=
 go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA=
+go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022 h1:8TRpo0lYh1Y6zec8Px0yXbnVRXXs+yUPU+TnHNsREdA=
+go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

internal/devconfig/env.go

@@ -1,15 +1,20 @@
 package devconfig
 
 import (
+	"context"
+
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/integrations/envsec"
 )
 
-func (c *Config) ComputedEnv(projectDir string) (map[string]string, error) {
+func (c *Config) ComputedEnv(
+	ctx context.Context,
+	projectDir string,
+) (map[string]string, error) {
 	env := map[string]string{}
 	var err error
 	if c.IsEnvsecEnabled() {
-		env, err = envsec.Env(projectDir)
+		env, err = envsec.Env(ctx, projectDir)
 		if err != nil {
 			return nil, err
 		}

internal/devpkg/pkgtype/runx.go

@@ -1,12 +1,39 @@
 package pkgtype
 
-import "strings"
+import (
+	"context"
+	"os"
+	"strings"
+
+	"go.jetpack.io/pkg/sandbox/runx/impl/registry"
+	"go.jetpack.io/pkg/sandbox/runx/impl/runx"
+)
 
 const (
-	RunXScheme = "runx"
-	RunXPrefix = RunXScheme + ":"
+	RunXScheme            = "runx"
+	RunXPrefix            = RunXScheme + ":"
+	githubAPITokenVarName = "DEVBOX_GITHUB_API_TOKEN"
 )
 
+var cachedRegistry *registry.Registry
+
 func IsRunX(s string) bool {
 	return strings.HasPrefix(s, RunXPrefix)
 }
+
+func RunXClient() *runx.RunX {
+	return &runx.RunX{
+		GithubAPIToken: os.Getenv(githubAPITokenVarName),
+	}
+}
+
+func RunXRegistry(ctx context.Context) (*registry.Registry, error) {
+	if cachedRegistry == nil {
+		var err error
+		cachedRegistry, err = registry.NewLocalRegistry(ctx, os.Getenv(githubAPITokenVarName))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return cachedRegistry, nil
+}

internal/devpkg/validation.go

@@ -1,16 +1,17 @@
 package devpkg
 
 import (
+	"context"
 	"strings"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/nix"
 )
 
-func (p *Package) ValidateExists() (bool, error) {
+func (p *Package) ValidateExists(ctx context.Context) (bool, error) {
 	if p.IsRunX() {
-		// TODO implement runx validation
-		return true, nil
+		_, err := p.lockfile.Resolve(p.Raw)
+		return err == nil, err
 	}
 	if p.isVersioned() && p.version() == "" {
 		return false, usererr.New("No version specified for %q.", p.Path)

internal/impl/devbox.go

@@ -22,12 +22,12 @@ import (
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devpkg"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/impl/envpath"
 	"go.jetpack.io/devbox/internal/impl/generate"
 	"go.jetpack.io/devbox/internal/searcher"
 	"go.jetpack.io/devbox/internal/shellgen"
 	"go.jetpack.io/devbox/internal/telemetry"
-	"go.jetpack.io/pkg/sandbox/runx"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cmdutil"
@@ -842,13 +842,13 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 		)
 	}
 
-	env["PATH"], err = d.addUtilitiesToPath(env["PATH"])
+	env["PATH"], err = d.addUtilitiesToPath(ctx, env["PATH"])
 	if err != nil {
 		return nil, err
 	}
 
 	// Include env variables in devbox.json
-	configEnv, err := d.configEnvs(env)
+	configEnv, err := d.configEnvs(ctx, env)
 	if err != nil {
 		return nil, err
 	}
@@ -876,7 +876,7 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 	})
 	debug.Log("PATH after filtering with buildInputs (%v) is: %s", buildInputs, nixEnvPath)
 
-	runXPaths, err := d.RunXPaths()
+	runXPaths, err := d.RunXPaths(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -1044,8 +1044,11 @@ func (d *Devbox) checkOldEnvrc() error {
 // their value in the existing env variables. Note, this doesn't
 // allow env variables from outside the shell to be referenced so
 // no leaked variables are caused by this function.
-func (d *Devbox) configEnvs(existingEnv map[string]string) (map[string]string, error) {
-	env, err := d.cfg.ComputedEnv(d.ProjectDir())
+func (d *Devbox) configEnvs(
+	ctx context.Context,
+	existingEnv map[string]string,
+) (map[string]string, error) {
+	env, err := d.cfg.ComputedEnv(ctx, d.ProjectDir())
 	if err != nil {
 		return nil, err
 	}
@@ -1206,11 +1209,15 @@ func (d *Devbox) Lockfile() *lock.File {
 	return d.lockfile
 }
 
-func (d *Devbox) RunXPaths() (string, error) {
+func (d *Devbox) RunXPaths(ctx context.Context) (string, error) {
 	packages := lo.Filter(d.InstallablePackages(), devpkg.IsRunX)
 	paths := []string{}
 	for _, pkg := range packages {
-		p, err := runx.Install(pkg.RunXPath())
+		lockedPkg, err := d.lockfile.Resolve(pkg.Raw)
+		if err != nil {
+			return "", err
+		}
+		p, err := pkgtype.RunXClient().Install(ctx, lockedPkg.Resolved)
 		if err != nil {
 			return "", err
 		}

internal/impl/packages.go

@@ -16,9 +16,9 @@ import (
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devpkg"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/nix/nixprofile"
 	"go.jetpack.io/devbox/internal/shellgen"
-	"go.jetpack.io/pkg/sandbox/runx"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
@@ -78,7 +78,7 @@ func (d *Devbox) Add(ctx context.Context, platforms, excludePlatforms []string,
 		versionedPkg := devpkg.PackageFromString(pkg.Versioned(), d.lockfile)
 
 		packageNameForConfig := pkg.Raw
-		ok, err := versionedPkg.ValidateExists()
+		ok, err := versionedPkg.ValidateExists(ctx)
 		if (err == nil && ok) || errors.Is(err, devpkg.ErrCannotBuildPackageOnSystem) {
 			// Only use versioned if it exists in search. We can disregard the error
 			// about not building on the current system, since user's can continue
@@ -236,7 +236,7 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod
 		return err
 	}
 
-	if err := d.InstallRunXPackages(); err != nil {
+	if err := d.InstallRunXPackages(ctx); err != nil {
 		return err
 	}
 
@@ -473,18 +473,17 @@ func resetProfileDirForFlakes(profileDir string) (err error) {
 	return errors.WithStack(os.Remove(profileDir))
 }
 
-func (d *Devbox) InstallRunXPackages() error {
-	for _, pkg := range d.InstallablePackages() {
-		if pkg.IsRunX() {
-			// TODO: Once resolve is implemented, we use whatever version is in the lockfile.
-			if _, err := d.lockfile.Resolve(pkg.Raw); err != nil {
-				return err
-			}
-			_, err := runx.Install(pkg.RunXPath())
-			if err != nil {
-				return fmt.Errorf("error installing runx package %s: %w", pkg, err)
-			}
-
+func (d *Devbox) InstallRunXPackages(ctx context.Context) error {
+	for _, pkg := range lo.Filter(d.InstallablePackages(), devpkg.IsRunX) {
+		lockedPkg, err := d.lockfile.Resolve(pkg.Raw)
+		if err != nil {
+			return err
+		}
+		if _, err := pkgtype.RunXClient().Install(
+			ctx,
+			lockedPkg.Resolved,
+		); err != nil {
+			return fmt.Errorf("error installing runx package %s: %w", pkg, err)
 		}
 	}
 	return nil

internal/impl/util.go

@@ -38,9 +38,12 @@ func (d *Devbox) addDevboxUtilityPackage(ctx context.Context, pkg string) error
 // to (e.g. envsec).
 // Question: Should we add utilityBinPath here? That would allow user to use
 // process-compose, etc
-func (d *Devbox) addUtilitiesToPath(path string) (string, error) {
+func (d *Devbox) addUtilitiesToPath(
+	ctx context.Context,
+	path string,
+) (string, error) {
 	if d.cfg.IsEnvsecEnabled() {
-		envsecPath, err := envsec.EnsureInstalled()
+		envsecPath, err := envsec.EnsureInstalled(ctx)
 		if err != nil {
 			return "", err
 		}

internal/integrations/envsec/envsec.go

@@ -2,39 +2,40 @@ package envsec
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
-	"go.jetpack.io/pkg/sandbox/runx"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 )
 
 var (
 	envCache     map[string]string
 	binPathCache string
 )
 
-func Env(projectDir string) (map[string]string, error) {
+func Env(ctx context.Context, projectDir string) (map[string]string, error) {
 	defer debug.FunctionTimer().End()
 
 	if envCache != nil {
 		return envCache, nil
 	}
 
-	if err := ensureInitialized(projectDir); err != nil {
+	if err := ensureInitialized(ctx, projectDir); err != nil {
 		return nil, err
 	}
 
 	var err error
-	envCache, err = envsecList(projectDir)
+	envCache, err = envsecList(ctx, projectDir)
 
 	return envCache, err
 }
 
-func EnsureInstalled() (string, error) {
+func EnsureInstalled(ctx context.Context) (string, error) {
 	if binPathCache != "" {
 		return binPathCache, nil
 	}
@@ -44,7 +45,7 @@ func EnsureInstalled() (string, error) {
 		return binPathCache, nil
 	}
 
-	paths, err := runx.Install("jetpack-io/envsec")
+	paths, err := pkgtype.RunXClient().Install(ctx, "jetpack-io/envsec")
 	if err != nil {
 		return "", errors.Wrap(err, "failed to install envsec")
 	}
@@ -57,8 +58,8 @@ func EnsureInstalled() (string, error) {
 	return binPathCache, nil
 }
 
-func ensureInitialized(projectDir string) error {
-	binPath, err := EnsureInstalled()
+func ensureInitialized(ctx context.Context, projectDir string) error {
+	binPath, err := EnsureInstalled(ctx)
 	if err != nil {
 		return err
 	}
@@ -73,8 +74,11 @@ func ensureInitialized(projectDir string) error {
 	return nil
 }
 
-func envsecList(projectDir string) (map[string]string, error) {
-	binPath, err := EnsureInstalled()
+func envsecList(
+	ctx context.Context,
+	projectDir string,
+) (map[string]string, error) {
+	binPath, err := EnsureInstalled(ctx)
 	if err != nil {
 		return nil, err
 	}

internal/lock/lockfile.go

@@ -4,6 +4,7 @@
 package lock
 
 import (
+	"context"
 	"fmt"
 	"io/fs"
 	"path/filepath"
@@ -13,6 +14,7 @@ import (
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/searcher"
+	"go.jetpack.io/pkg/sandbox/runx/impl/types"
 
 	"go.jetpack.io/devbox/internal/cuecfg"
 )
@@ -71,9 +73,13 @@ func (f *File) Resolve(pkg string) (*Package, error) {
 		locked := &Package{}
 		var err error
 		if pkgtype.IsRunX(pkg) {
-			// TODO implement runx resolution. This can be done by reading the releases.json file
+			ref, err := ResolveRunXPackage(context.TODO(), pkg)
+			if err != nil {
+				return nil, err
+			}
 			locked = &Package{
-				Resolved: pkg,
+				Resolved: ref.String(),
+				Version:  ref.Version,
 			}
 		} else if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned {
 			locked, err = f.FetchResolvedPackage(pkg)
@@ -185,3 +191,16 @@ func lockFilePath(project devboxProject) string {
 func getLockfileHash(project devboxProject) (string, error) {
 	return cuecfg.FileHash(lockFilePath(project))
 }
+
+func ResolveRunXPackage(ctx context.Context, pkg string) (types.PkgRef, error) {
+	ref, err := types.NewPkgRef(strings.TrimPrefix(pkg, pkgtype.RunXPrefix))
+	if err != nil {
+		return types.PkgRef{}, err
+	}
+
+	registry, err := pkgtype.RunXRegistry(ctx)
+	if err != nil {
+		return types.PkgRef{}, err
+	}
+	return registry.ResolveVersion(ref)
+}