[remove nixpkgs] add toPath, and edit devbox.lock only on update command

Author: savil

devbox.json

@@ -1,7 +1,6 @@
 {
   "packages": [
     "[email protected]",
-    "[email protected]",
     "[email protected]"
   ],
   "env": {

devbox.lock

@@ -1,11 +1,6 @@
 {
   "lockfile_version": "1",
   "packages": {
-    "[email protected]": {
-      "last_modified": "2023-03-31T22:52:29Z",
-      "resolved": "github:NixOS/nixpkgs/242246ee1e58f54d2322227fc5eef53b4a616a31#actionlint",
-      "version": "1.6.23"
-    },
     "[email protected]": {
       "last_modified": "2023-05-25T03:54:59Z",
       "resolved": "github:NixOS/nixpkgs/8d4d822bc0efa9de6eddc79cb0d82897a9baa750#go",

internal/devpkg/package.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -19,6 +20,7 @@ import (
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/nix"
+	"go.jetpack.io/devbox/internal/ux"
 )
 
 // Package represents a "package" added to the devbox.json config.
@@ -432,7 +434,7 @@ func (p *Package) IsInBinaryStore() (bool, error) {
 }
 
 // PathInBinaryStore is the key in the BinaryCacheStore for this package
-// This is used as FromPath in builtins.fetchClosure
+// This is used as BinaryStorePath in builtins.fetchClosure
 func (p *Package) PathInBinaryStore() (string, error) {
 	if isInStore, err := p.IsInBinaryStore(); err != nil {
 		return "", err
@@ -452,10 +454,40 @@ func (p *Package) PathInBinaryStore() (string, error) {
 	}
 
 	sysInfo := entry.Systems[userSystem]
-	storeDirParts := []string{sysInfo.FromHash, sysInfo.StoreName}
-	if sysInfo.StoreVersion != "" {
-		storeDirParts = append(storeDirParts, sysInfo.StoreVersion)
+	return sysInfo.BinaryStorePath, nil
+}
+
+func (p *Package) PathInLocalStore() (string, error) {
+
+	if isInStore, err := p.IsInBinaryStore(); err != nil {
+		return "", err
+	} else if !isInStore {
+		return "",
+			errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw)
+	}
+
+	entry, err := p.lockfile.Resolve(p.Raw)
+	if err != nil {
+		return "", err
+	}
+
+	userSystem, err := nix.System()
+	if err != nil {
+		return "", err
+	}
+
+	sysInfo := entry.Systems[userSystem]
+	if sysInfo.LocalStorePath != "" {
+		return sysInfo.LocalStorePath, nil
+	}
+
+	ux.Fwarning(
+		os.Stderr, 
+		"calculating local_store_path. This may be slow.\n" +
+		"Run `devbox update` to speed this up for next time.")
+	localPath, err := nix.ContentAddressedStorePath(sysInfo.BinaryStorePath)
+	if err != nil {
+		return "", err
 	}
-	storeDir := strings.Join(storeDirParts, "-")
-	return filepath.Join("/nix/store", storeDir), nil
+	return localPath, err
 }

internal/impl/update.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/nix/nixprofile"
@@ -90,21 +91,52 @@ func (d *Devbox) updateDevboxPackage(
 	if err != nil {
 		return err
 	}
-	if existing != nil && existing.Version != newEntry.Version {
+	if existing == nil {
+		ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved)
+		d.lockfile.Packages[pkg.Raw] = newEntry
+		return nil
+	}
+
+	if existing.Version != newEntry.Version {
 		ux.Finfo(d.writer, "Updating %s %s -> %s\n", pkg, existing.Version, newEntry.Version)
 		if err := d.removePackagesFromProfile(ctx, []string{pkg.Raw}); err != nil {
 			// Warn but continue. TODO(landau): ensurePackagesAreInstalled should
 			// sync the profile so we don't need to do this manually.
 			ux.Fwarning(d.writer, "Failed to remove %s from profile: %s\n", pkg, err)
 		}
 		d.lockfile.Packages[pkg.Raw] = newEntry
-	} else if existing == nil {
-		ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved)
+		return nil
+	}
+
+	// Check if the package's system info is missing, or not complete.
+	userSystem, err := nix.System()
+	if err != nil {
+		return err
+	}
+	var needsSysInfo bool
+	var needsLocalStorePath bool
+	if featureflag.RemoveNixpkgs.Enabled() {
+		userSysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem]
+		needsSysInfo = userSysInfo == nil
+		if !needsSysInfo {
+			// Check if the LocalStorePath is missing for the user's system.
+			// Since any one user cannot add this field for all systems, 
+			// we'll need to progressively add it to a project's lockfile.
+			needsLocalStorePath = userSysInfo.LocalStorePath == ""
+		}
+	}
+	if needsSysInfo {
 		d.lockfile.Packages[pkg.Raw] = newEntry
-	} else {
-		ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version)
+		ux.Finfo(d.writer, "Updated system information for %s\n", pkg)
+		return nil
+	} else if needsLocalStorePath {
+		// Update the LocalStorePath for the user's system
+		d.lockfile.Packages[pkg.Raw].Systems[userSystem].LocalStorePath = newEntry.Systems[userSystem].LocalStorePath
+		ux.Finfo(d.writer, "Updated system information for %s\n", pkg)
+		return nil
 	}
 
+	ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version)
 	return nil
 }
 

internal/lock/lockfile.go

@@ -11,8 +11,6 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
-	"go.jetpack.io/devbox/internal/boxcli/featureflag"
-	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/searcher"
 
 	"go.jetpack.io/devbox/internal/cuecfg"
@@ -40,12 +38,13 @@ type Package struct {
 }
 
 type SystemInfo struct {
-	System   string // stored elsewhere in json: it's the key for the Package.Systems
-	FromHash string `json:"from_hash,omitempty"`
-	// StoreName may be different from the canonicalName so we store it separately
-	StoreName    string `json:"store_name,omitempty"`
-	StoreVersion string `json:"store_version,omitempty"`
-	ToHash       string `json:"to_hash,omitempty"`
+	// BinaryStorePath is the cache key in the Binary Cache Store (cache.nixos.org)
+	// It is of the form <hash>-<name>-<version>
+	// <name> may be different from the canonicalName so we store the full store path.
+	BinaryStorePath string `json:"bin_store_path,omitempty"`
+	// LocalStorePath is the content-addressed path for the nix package in /nix/store
+	// It is of the form <hash>-<name>-<version>
+	LocalStorePath string `json:"local_store_path,omitempty"`
 }
 
 func GetFile(project devboxProject) (*File, error) {
@@ -86,17 +85,7 @@ func (l *File) Remove(pkgs ...string) error {
 func (l *File) Resolve(pkg string) (*Package, error) {
 	entry, hasEntry := l.Packages[pkg]
 
-	// If the package's system info is missing, we need to resolve it again.
-	needsSysInfo := false
-	if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
-		userSystem, err := nix.System()
-		if err != nil {
-			return nil, err
-		}
-		needsSysInfo = entry.Systems[userSystem] == nil
-	}
-
-	if !hasEntry || entry.Resolved == "" || needsSysInfo {
+	if !hasEntry || entry.Resolved == "" {
 		locked := &Package{}
 		var err error
 		if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned {
@@ -109,19 +98,6 @@ func (l *File) Resolve(pkg string) (*Package, error) {
 			// whatever hash is in the devbox.json
 			locked = &Package{Resolved: l.LegacyNixpkgsPath(pkg)}
 		}
-
-		// Merge the system info from the lockfile with the queried system info.
-		// This is necessary because a different system's info may previously have
-		// been added. For example: `aarch64-darwin` was already added, but
-		// current user is on `x86_64-linux`.
-		if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
-			for _, sysInfo := range entry.Systems {
-				if _, ok := locked.Systems[sysInfo.System]; !ok {
-					locked.Systems[sysInfo.System] = sysInfo
-				}
-			}
-		}
-
 		l.Packages[pkg] = locked
 	}
 

internal/lock/resolve.go

@@ -34,7 +34,10 @@ func (l *File) FetchResolvedPackage(pkg string) (*Package, error) {
 
 	sysInfos := map[string]*SystemInfo{}
 	if featureflag.RemoveNixpkgs.Enabled() {
-		sysInfos = buildLockSystemInfos(packageVersion)
+		sysInfos, err = buildLockSystemInfos(packageVersion)
+		if err != nil {
+			return nil, err
+		}
 	}
 	packageInfo, err := selectForSystem(packageVersion)
 	if err != nil {
@@ -76,15 +79,32 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error)
 	return maps.Values(pkg.Systems)[0], nil
 }
 
-func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo {
+func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) {
+	userSystem, err := nix.System()
+	if err != nil {
+		return nil, err
+	}
+
 	sysInfos := map[string]*SystemInfo{}
 	for sysName, sysInfo := range pkg.Systems {
+		
+		// guard against missing search data
+		if sysInfo.StoreHash == "" || sysInfo.StoreName == "" {
+			continue
+		}
+
+		localStorePath := ""
+		if sysName == userSystem {
+			binaryStorePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion)
+			localStorePath, err = nix.ContentAddressedStorePath(binaryStorePath)
+			if err != nil {
+				return nil, err
+			}
+		}
 		sysInfos[sysName] = &SystemInfo{
-			System:       sysName,
-			FromHash:     sysInfo.StoreHash,
-			StoreName:    sysInfo.StoreName,
-			StoreVersion: sysInfo.StoreVersion,
+			BinaryStorePath: nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion),
+			LocalStorePath:  localStorePath,
 		}
 	}
-	return sysInfos
+	return sysInfos, nil
 }

internal/nix/nix.go

@@ -66,9 +66,6 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 			args.FlakesFilePath,
 		)
 		cmd.Args = append(cmd.Args, ExperimentalFlags()...)
-		if featureflag.RemoveNixpkgs.Enabled() {
-			cmd.Args = append(cmd.Args, "--impure")
-		}
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
 		data, err = cmd.Output()

internal/nix/store.go

@@ -0,0 +1,42 @@
+package nix
+
+import (
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+func StorePath(hash, name, version string) string {
+	storeDirParts := []string{hash, name}
+	if version != "" {
+		storeDirParts = append(storeDirParts, version)
+	}
+	storeDir := strings.Join(storeDirParts, "-")
+	return filepath.Join("/nix/store", storeDir)
+}
+
+// contentAddressedRegex matches the output of `nix store make-content-addressed`.
+// It is used to select the content-addressed store path (the second one in the example below).
+//
+// Example:
+// > nix store make-content-addressed  /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
+// rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
+var contentAddressedRegex = regexp.MustCompile(`rewrote\s'[\/a-z0-9-\.]+'\sto\s'([a-z0-9-\/\.]+)'`)
+
+// ContentAddressedStorePath takes a store path and returns the content-addressed store path.
+func ContentAddressedStorePath(storePath string) (string, error) {
+	cmd := command("store", "make-content-addressed", storePath)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	matches := contentAddressedRegex.FindStringSubmatch(string(out))
+	if len(matches) < 2 {
+		return "", errors.Errorf("could not parse output of nix store make-content-addressed: %s", string(out))
+	}
+
+	return matches[1], nil
+}

internal/nix/store_test.go

@@ -0,0 +1,32 @@
+package nix
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestContentAddressedPath(t *testing.T) {
+
+	testCases := []struct {
+		storePath string
+		expected  string
+	}{
+		{
+			"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1",
+			"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1",
+		},
+	}
+
+	for index, testCase := range testCases {
+		t.Run(fmt.Sprintf("%d", index), func(t *testing.T) {
+			out, err := ContentAddressedStorePath(testCase.storePath)
+			if err != nil {
+				t.Errorf("got error: %v", err)
+			}
+			if out != testCase.expected {
+				t.Errorf("got %s, want %s", out, testCase.expected)
+			}
+		})
+
+	}
+}

internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

@@ -26,6 +26,7 @@
             (builtins.fetchClosure{
               fromStore = "{{ $.BinaryCacheStore }}";
               fromPath = "{{ .PathInBinaryStore }}";
+              toPath = "{{ .PathInLocalStore }}";
             })
             {{- end }}
             {{- end }}