fixes: experimental flags, UX improvements

Author: savil

internal/devbox/nixprofile.go

@@ -4,54 +4,79 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os/exec"
 	"slices"
+	"strings"
 
 	"go.jetpack.io/devbox/internal/nix"
+	"go.jetpack.io/devbox/internal/nix/nixprofile"
 )
 
-func syncFlakeToProfile(ctx context.Context, flakePath, profilePath string) error {
-	cmd := exec.CommandContext(ctx, "nix", "eval", "--json", flakePath+"#devShells."+nix.System()+".default.buildInputs")
-	b, err := cmd.Output()
+func (d *Devbox) syncFlakeToProfile(ctx context.Context) error {
+	profilePath, err := d.profilePath()
+	if err != nil {
+		return err
+	}
+
+	// Get the build inputs (i.e. store paths) from the generated flake's devShell.
+	buildInputPaths, err := nix.Eval(
+		ctx,
+		d.stderr,
+		d.flakeDir()+"#devShells."+nix.System()+".default.buildInputs",
+		"--json",
+	)
 	if err != nil {
 		return fmt.Errorf("nix eval devShells: %v", err)
 	}
 	storePaths := []string{}
-	if err := json.Unmarshal(b, &storePaths); err != nil {
-		return fmt.Errorf("unmarshal store paths: %s: %v", b, err)
+	if err := json.Unmarshal(buildInputPaths, &storePaths); err != nil {
+		return fmt.Errorf("unmarshal store paths: %s: %v", buildInputPaths, err)
 	}
 
-	listCmd := exec.CommandContext(ctx, "nix", "profile", "list", "--json", "--profile", profilePath)
-	b, err = listCmd.Output()
+	// Get the store-paths of the packages currently installed in the nix profile
+	items, err := nixprofile.ProfileListItems(ctx, d.stderr, profilePath)
 	if err != nil {
-		return err
-	}
-	var profile struct {
-		Elements []struct {
-			StorePaths []string
-		}
+		return fmt.Errorf("nix profile list: %v", err)
 	}
-	if err := json.Unmarshal(b, &profile); err != nil {
-		return fmt.Errorf("unmarshal profile: %v", err)
-	}
-	got := make([]string, 0, len(profile.Elements))
-	for _, e := range profile.Elements {
-		got = append(got, e.StorePaths...)
+	got := make([]string, 0, len(items))
+	for _, item := range items {
+		got = append(got, item.StorePaths()...)
 	}
 
+	// Diff the store paths and install/remove packages as needed
 	add, remove := diffStorePaths(got, storePaths)
 	if len(remove) > 0 {
-		removeCmd := exec.CommandContext(ctx, "nix", "profile", "remove", "--profile", profilePath)
-		removeCmd.Args = append(removeCmd.Args, remove...)
-		if err := removeCmd.Run(); err != nil {
+		packagesToRemove := make([]string, 0, len(remove))
+		for _, p := range remove {
+			storePath := nix.NewStorePathParts(p)
+			packagesToRemove = append(packagesToRemove, fmt.Sprintf("%s@%s", storePath.Name, storePath.Version))
+		}
+		if len(packagesToRemove) == 1 {
+			fmt.Fprintf(d.stderr, "Removing %s\n", strings.Join(packagesToRemove, ", "))
+		} else {
+			fmt.Fprintf(d.stderr, "Removing packages: %s\n", strings.Join(packagesToRemove, ", "))
+		}
+
+		if err := nix.ProfileRemove(profilePath, remove...); err != nil {
 			return err
 		}
 	}
 	if len(add) > 0 {
-		addCmd := exec.CommandContext(ctx, "nix", "profile", "install", "--profile", profilePath)
-		addCmd.Args = append(addCmd.Args, add...)
-		if err := addCmd.Run(); err != nil {
-			return err
+		total := len(add)
+		for idx, addPath := range add {
+			stepNum := idx + 1
+			storePath := nix.NewStorePathParts(addPath)
+			nameAndVersion := fmt.Sprintf("%s@%s", storePath.Name, storePath.Version)
+			stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, nameAndVersion)
+
+			if err = nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{
+				CustomStepMessage: stepMsg,
+				Installable:       addPath,
+				PackageName:       storePath.Name,
+				ProfilePath:       profilePath,
+				Writer:            d.stderr,
+			}); err != nil {
+				return fmt.Errorf("error installing package %s: %w", addPath, err)
+			}
 		}
 	}
 	return nil
@@ -61,27 +86,27 @@ func diffStorePaths(got, want []string) (add, remove []string) {
 	slices.Sort(got)
 	slices.Sort(want)
 
-	var g, w int
+	var gotIdx, wantIdx int
 	for {
-		if g >= len(got) {
-			add = append(add, want[w:]...)
+		if gotIdx >= len(got) {
+			add = append(add, want[wantIdx:]...)
 			break
 		}
-		if w >= len(want) {
-			remove = append(remove, got[g:]...)
+		if wantIdx >= len(want) {
+			remove = append(remove, got[gotIdx:]...)
 			break
 		}
 
 		switch {
-		case got[g] == want[w]:
-			g++
-			w++
-		case got[g] < want[w]:
-			remove = append(remove, got[g])
-			g++
-		case got[g] > want[w]:
-			add = append(add, want[w])
-			w++
+		case got[gotIdx] == want[wantIdx]:
+			gotIdx++
+			wantIdx++
+		case got[gotIdx] < want[wantIdx]:
+			remove = append(remove, got[gotIdx])
+			gotIdx++
+		case got[gotIdx] > want[wantIdx]:
+			add = append(add, want[wantIdx])
+			wantIdx++
 		}
 	}
 	return add, remove

internal/devbox/packages.go

@@ -292,11 +292,7 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er
 		return err
 	}
 
-	profile, err := d.profilePath()
-	if err != nil {
-		return err
-	}
-	if err := syncFlakeToProfile(ctx, d.flakeDir(), profile); err != nil {
+	if err := d.syncFlakeToProfile(ctx); err != nil {
 		return err
 	}
 

internal/devbox/util.go

@@ -26,7 +26,7 @@ func (d *Devbox) addDevboxUtilityPackage(ctx context.Context, pkg string) error
 		return err
 	}
 
-	return nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{
+	return nixprofile.ProfileInstallPackage(ctx, &nixprofile.ProfileInstallPackageArgs{
 		Lockfile:    d.lockfile,
 		Package:     pkg,
 		ProfilePath: profilePath,

internal/devpkg/narinfo_cache.go

@@ -4,10 +4,8 @@ import (
 	"context"
 	"io"
 	"net/http"
-	"strings"
 	"sync"
 	"time"
-	"unicode"
 
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
@@ -110,8 +108,8 @@ func (p *Package) fetchNarInfoStatus() (bool, error) {
 		)
 	}
 
-	pathParts := newStorePathParts(sysInfo.StorePath)
-	reqURL := BinaryCache + "/" + pathParts.hash + ".narinfo"
+	pathParts := nix.NewStorePathParts(sysInfo.StorePath)
+	reqURL := BinaryCache + "/" + pathParts.Hash + ".narinfo"
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, reqURL, nil)
@@ -181,35 +179,3 @@ func (p *Package) sysInfoIfExists() (*lock.SystemInfo, error) {
 	}
 	return sysInfo, nil
 }
-
-// storePath are the constituent parts of
-// /nix/store/<hash>-<name>-<version>
-//
-// This is a helper struct for analyzing the string representation
-type storePathParts struct {
-	hash    string
-	name    string
-	version string
-}
-
-// newStorePathParts splits a Nix store path into its hash, name and version
-// components in the same way that Nix does.
-//
-// See https://nixos.org/manual/nix/stable/language/builtins.html#builtins-parseDrvName
-func newStorePathParts(path string) storePathParts {
-	path = strings.TrimPrefix(path, "/nix/store/")
-	// path is now <hash>-<name>-<version
-
-	hash, name := path[:32], path[33:]
-	dashIndex := 0
-	for i, r := range name {
-		if dashIndex != 0 && !unicode.IsLetter(r) {
-			return storePathParts{hash: hash, name: name[:dashIndex], version: name[i:]}
-		}
-		dashIndex = 0
-		if r == '-' {
-			dashIndex = i
-		}
-	}
-	return storePathParts{hash: hash, name: name}
-}

internal/devpkg/package_test.go

@@ -181,49 +181,6 @@ func TestHashFromNixPkgsURL(t *testing.T) {
 	}
 }
 
-func TestStorePathParts(t *testing.T) {
-	testCases := []struct {
-		storePath string
-		expected  storePathParts
-	}{
-		// simple case:
-		{
-			storePath: "/nix/store/cvrn84c1hshv2wcds7n1rhydi6lacqns-gnumake-4.4.1",
-			expected: storePathParts{
-				hash:    "cvrn84c1hshv2wcds7n1rhydi6lacqns",
-				name:    "gnumake",
-				version: "4.4.1",
-			},
-		},
-		// the package name can have dashes:
-		{
-			storePath: "/nix/store/q2xdxsswjqmqcbax81pmazm367s7jzyb-cctools-binutils-darwin-wrapper-973.0.1",
-			expected: storePathParts{
-				hash:    "q2xdxsswjqmqcbax81pmazm367s7jzyb",
-				name:    "cctools-binutils-darwin-wrapper",
-				version: "973.0.1",
-			},
-		},
-		// version is optional. This is an artificial example I constructed
-		{
-			storePath: "/nix/store/gfxwrd5nggc68pjj3g3jhlldim9rpg0p-coreutils",
-			expected: storePathParts{
-				hash: "gfxwrd5nggc68pjj3g3jhlldim9rpg0p",
-				name: "coreutils",
-			},
-		},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.storePath, func(t *testing.T) {
-			parts := newStorePathParts(testCase.storePath)
-			if parts != testCase.expected {
-				t.Errorf("Expected %v, got %v", testCase.expected, parts)
-			}
-		})
-	}
-}
-
 func TestCanonicalName(t *testing.T) {
 	tests := []struct {
 		pkgName      string

internal/devpkg/validation.go

@@ -38,7 +38,7 @@ func (p *Package) ValidateInstallsOnSystem() (bool, error) {
 	if len(info) == 0 {
 		return false, nil
 	}
-	if out, err := nix.Eval(u); err != nil &&
+	if out, err := nix.RawEval(u); err != nil &&
 		strings.Contains(string(out), "is not available on the requested hostPlatform") {
 		return false, nil
 	}

internal/nix/eval.go

@@ -1,7 +1,9 @@
 package nix
 
 import (
+	"context"
 	"encoding/json"
+	"io"
 	"os"
 	"strconv"
 )
@@ -46,14 +48,22 @@ func PackageKnownVulnerabilities(path string) []string {
 	return vulnerabilities
 }
 
-// Eval is raw nix eval. Needs to be parsed. Useful for stuff like
+// RawEval is nix eval. Needs to be parsed. Useful for stuff like
 // nix eval --raw nixpkgs/9ef09e06806e79e32e30d17aee6879d69c011037#fuse3
 // to determine if a package if a package can be installed in system.
-func Eval(path string) ([]byte, error) {
+func RawEval(path string) ([]byte, error) {
 	cmd := command("eval", "--raw", path)
 	return cmd.CombinedOutput()
 }
 
+func Eval(ctx context.Context, w io.Writer, installable string, options ...string) ([]byte, error) {
+	args := []string{"eval", installable}
+	args = append(args, options...)
+	cmd := commandContext(ctx, args...)
+	cmd.Stderr = w
+	return cmd.Output()
+}
+
 func AllowInsecurePackages() {
 	os.Setenv("NIXPKGS_ALLOW_INSECURE", "1")
 }

internal/nix/nixprofile/item.go

@@ -83,3 +83,7 @@ func (i *NixProfileListItem) String() string {
 		i.nixStorePaths,
 	)
 }
+
+func (i *NixProfileListItem) StorePaths() []string {
+	return i.nixStorePaths
+}