Skip to content

Commit 9a249d3

Browse files
savilsavil
committed
[remove nixpkgs] add toPath, and edit devbox.lock only on update command
1 parent 6c833de commit 9a249d3

File tree

8 files changed

+181
-45
lines changed

8 files changed

+181
-45
lines changed

‎internal/devpkg/package.go

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ func (p *Package) IsInBinaryStore() (bool, error) {
432432
}
433433

434434
// PathInBinaryStore is the key in the BinaryCacheStore for this package
435-
// This is used as FromPath in builtins.fetchClosure
435+
// This is used as BinaryStorePath in builtins.fetchClosure
436436
func (p *Package) PathInBinaryStore() (string, error) {
437437
if isInStore, err := p.IsInBinaryStore(); err != nil {
438438
return "", err
@@ -452,10 +452,28 @@ func (p *Package) PathInBinaryStore() (string, error) {
452452
}
453453

454454
sysInfo := entry.Systems[userSystem]
455-
storeDirParts := []string{sysInfo.FromHash, sysInfo.StoreName}
456-
if sysInfo.StoreVersion != "" {
457-
storeDirParts = append(storeDirParts, sysInfo.StoreVersion)
455+
return sysInfo.BinaryStorePath, nil
456+
}
457+
458+
func (p *Package) PathInLocalStore() (string, error) {
459+
460+
if isInStore, err := p.IsInBinaryStore(); err != nil {
461+
return "", err
462+
} else if !isInStore {
463+
return "",
464+
errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw)
465+
}
466+
467+
entry, err := p.lockfile.Resolve(p.Raw)
468+
if err != nil {
469+
return "", err
470+
}
471+
472+
userSystem, err := nix.System()
473+
if err != nil {
474+
return "", err
458475
}
459-
storeDir := strings.Join(storeDirParts, "-")
460-
return filepath.Join("/nix/store", storeDir), nil
476+
477+
sysInfo := entry.Systems[userSystem]
478+
return sysInfo.LocalStorePath, nil
461479
}

‎internal/impl/update.go

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"context"
88
"fmt"
99

10+
"go.jetpack.io/devbox/internal/boxcli/featureflag"
1011
"go.jetpack.io/devbox/internal/devpkg"
1112
"go.jetpack.io/devbox/internal/nix"
1213
"go.jetpack.io/devbox/internal/nix/nixprofile"
@@ -90,21 +91,52 @@ func (d *Devbox) updateDevboxPackage(
9091
if err != nil {
9192
return err
9293
}
93-
if existing != nil && existing.Version != newEntry.Version {
94+
if existing == nil {
95+
ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved)
96+
d.lockfile.Packages[pkg.Raw] = newEntry
97+
return nil
98+
}
99+
100+
if existing.Version != newEntry.Version {
94101
ux.Finfo(d.writer, "Updating %s %s -> %s\n", pkg, existing.Version, newEntry.Version)
95102
if err := d.removePackagesFromProfile(ctx, []string{pkg.Raw}); err != nil {
96103
// Warn but continue. TODO(landau): ensurePackagesAreInstalled should
97104
// sync the profile so we don't need to do this manually.
98105
ux.Fwarning(d.writer, "Failed to remove %s from profile: %s\n", pkg, err)
99106
}
100107
d.lockfile.Packages[pkg.Raw] = newEntry
101-
} else if existing == nil {
102-
ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved)
108+
return nil
109+
}
110+
111+
// Check if the package's system info is missing, or not complete.
112+
userSystem, err := nix.System()
113+
if err != nil {
114+
return err
115+
}
116+
var needsSysInfo bool
117+
var needsLocalStorePath bool
118+
if featureflag.RemoveNixpkgs.Enabled() {
119+
userSysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem]
120+
needsSysInfo = userSysInfo == nil
121+
if !needsSysInfo {
122+
// Check if the LocalStorePath is missing for the user's system
123+
// Since any one user cannot add this field. We'll need to progressively
124+
// add it to a project's lockfile.
125+
needsLocalStorePath = userSysInfo.LocalStorePath == ""
126+
}
127+
}
128+
if needsSysInfo {
103129
d.lockfile.Packages[pkg.Raw] = newEntry
104-
} else {
105-
ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version)
130+
ux.Finfo(d.writer, "Updated system information for %s\n", pkg)
131+
return nil
132+
} else if needsLocalStorePath {
133+
// Update the LocalStorePath for the user's system
134+
d.lockfile.Packages[pkg.Raw].Systems[userSystem].LocalStorePath = newEntry.Systems[userSystem].LocalStorePath
135+
ux.Finfo(d.writer, "Updated system information for %s\n", pkg)
136+
return nil
106137
}
107138

139+
ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version)
108140
return nil
109141
}
110142

‎internal/lock/lockfile.go

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ import (
1111

1212
"github.com/pkg/errors"
1313
"github.com/samber/lo"
14-
"go.jetpack.io/devbox/internal/boxcli/featureflag"
15-
"go.jetpack.io/devbox/internal/nix"
1614
"go.jetpack.io/devbox/internal/searcher"
1715

1816
"go.jetpack.io/devbox/internal/cuecfg"
@@ -40,12 +38,14 @@ type Package struct {
4038
}
4139

4240
type SystemInfo struct {
43-
System string // stored elsewhere in json: it's the key for the Package.Systems
44-
FromHash string `json:"from_hash,omitempty"`
45-
// StoreName may be different from the canonicalName so we store it separately
46-
StoreName string `json:"store_name,omitempty"`
47-
StoreVersion string `json:"store_version,omitempty"`
48-
ToHash string `json:"to_hash,omitempty"`
41+
System string // stored elsewhere in json: it's the key for the Package.Systems
42+
// BinaryStorePath is the path to the derivation in the Binary Cache Store (cache.nixos.org)
43+
// It is of the form <hash>-<name>-<version>
44+
// <name> may be different from the canonicalName so we store the full store path.
45+
BinaryStorePath string `json:"bin_store_path,omitempty"`
46+
// LocalStorePath is the content-addressed path to the derivation
47+
// It is of the form <hash>-<name>-<version>
48+
LocalStorePath string `json:"local_store_path,omitempty"`
4949
}
5050

5151
func GetFile(project devboxProject) (*File, error) {
@@ -86,17 +86,7 @@ func (l *File) Remove(pkgs ...string) error {
8686
func (l *File) Resolve(pkg string) (*Package, error) {
8787
entry, hasEntry := l.Packages[pkg]
8888

89-
// If the package's system info is missing, we need to resolve it again.
90-
needsSysInfo := false
91-
if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
92-
userSystem, err := nix.System()
93-
if err != nil {
94-
return nil, err
95-
}
96-
needsSysInfo = entry.Systems[userSystem] == nil
97-
}
98-
99-
if !hasEntry || entry.Resolved == "" || needsSysInfo {
89+
if !hasEntry || entry.Resolved == "" {
10090
locked := &Package{}
10191
var err error
10292
if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned {
@@ -114,13 +104,16 @@ func (l *File) Resolve(pkg string) (*Package, error) {
114104
// This is necessary because a different system's info may previously have
115105
// been added. For example: `aarch64-darwin` was already added, but
116106
// current user is on `x86_64-linux`.
117-
if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
118-
for _, sysInfo := range entry.Systems {
119-
if _, ok := locked.Systems[sysInfo.System]; !ok {
120-
locked.Systems[sysInfo.System] = sysInfo
107+
/*
108+
if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
109+
for _, sysInfo := range entry.Systems {
110+
if _, ok := locked.Systems[sysInfo.System]; !ok {
111+
locked.Systems[sysInfo.System] = sysInfo
112+
}
121113
}
122114
}
123-
}
115+
116+
*/
124117

125118
l.Packages[pkg] = locked
126119
}

‎internal/lock/resolve.go

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@ func (l *File) FetchResolvedPackage(pkg string) (*Package, error) {
3434

3535
sysInfos := map[string]*SystemInfo{}
3636
if featureflag.RemoveNixpkgs.Enabled() {
37-
sysInfos = buildLockSystemInfos(packageVersion)
37+
sysInfos, err = buildLockSystemInfos(packageVersion)
38+
if err != nil {
39+
return nil, err
40+
}
3841
}
3942
packageInfo, err := selectForSystem(packageVersion)
4043
if err != nil {
@@ -76,15 +79,33 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error)
7679
return maps.Values(pkg.Systems)[0], nil
7780
}
7881

79-
func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo {
82+
func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) {
83+
userSystem, err := nix.System()
84+
if err != nil {
85+
return nil, err
86+
}
87+
8088
sysInfos := map[string]*SystemInfo{}
8189
for sysName, sysInfo := range pkg.Systems {
90+
91+
// guard against missing search data
92+
if sysInfo.StoreHash == "" || sysInfo.StoreName == "" {
93+
continue
94+
}
95+
96+
caPath := ""
97+
if sysName == userSystem {
98+
binaryStorePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion)
99+
caPath, err = nix.ContentAddressedStorePath(binaryStorePath)
100+
if err != nil {
101+
return nil, err
102+
}
103+
}
82104
sysInfos[sysName] = &SystemInfo{
83-
System: sysName,
84-
FromHash: sysInfo.StoreHash,
85-
StoreName: sysInfo.StoreName,
86-
StoreVersion: sysInfo.StoreVersion,
105+
System: sysName,
106+
BinaryStorePath: nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion),
107+
LocalStorePath: caPath,
87108
}
88109
}
89-
return sysInfos
110+
return sysInfos, nil
90111
}

‎internal/nix/nix.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,6 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
6666
args.FlakesFilePath,
6767
)
6868
cmd.Args = append(cmd.Args, ExperimentalFlags()...)
69-
if featureflag.RemoveNixpkgs.Enabled() {
70-
cmd.Args = append(cmd.Args, "--impure")
71-
}
7269
cmd.Args = append(cmd.Args, "--json")
7370
debug.Log("Running print-dev-env cmd: %s\n", cmd)
7471
data, err = cmd.Output()

‎internal/nix/store.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package nix
2+
3+
import (
4+
"path/filepath"
5+
"regexp"
6+
"strings"
7+
8+
"github.com/pkg/errors"
9+
)
10+
11+
func StorePath(hash, name, version string) string {
12+
storeDirParts := []string{hash, name}
13+
if version != "" {
14+
storeDirParts = append(storeDirParts, version)
15+
}
16+
storeDir := strings.Join(storeDirParts, "-")
17+
return filepath.Join("/nix/store", storeDir)
18+
}
19+
20+
// contentAddressedRegex matches the output of `nix store make-content-addressed`.
21+
// It is used to select the content-addressed store path (the second one in the example below).
22+
//
23+
// Example:
24+
// > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
25+
// rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
26+
var contentAddressedRegex = regexp.MustCompile(`rewrote\s'[\/a-z0-9-\.]+'\sto\s'([a-z0-9-\/\.]+)'`)
27+
28+
// ContentAddressedStorePath takes a store path and returns the content-addressed store path.
29+
func ContentAddressedStorePath(storePath string) (string, error) {
30+
cmd := command("store", "make-content-addressed", storePath)
31+
out, err := cmd.CombinedOutput()
32+
if err != nil {
33+
return "", errors.WithStack(err)
34+
}
35+
36+
matches := contentAddressedRegex.FindStringSubmatch(string(out))
37+
if len(matches) < 2 {
38+
return "", errors.Errorf("could not parse output of nix store make-content-addressed: %s", string(out))
39+
}
40+
41+
return matches[1], nil
42+
}

‎internal/nix/store_test.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package nix
2+
3+
import (
4+
"fmt"
5+
"testing"
6+
)
7+
8+
func TestContentAddressedPath(t *testing.T) {
9+
10+
testCases := []struct {
11+
storePath string
12+
expected string
13+
}{
14+
{
15+
"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1",
16+
"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1",
17+
},
18+
}
19+
20+
for index, testCase := range testCases {
21+
t.Run(fmt.Sprintf("%d", index), func(t *testing.T) {
22+
out, err := ContentAddressedStorePath(testCase.storePath)
23+
if err != nil {
24+
t.Errorf("got error: %v", err)
25+
}
26+
if out != testCase.expected {
27+
t.Errorf("got %s, want %s", out, testCase.expected)
28+
}
29+
})
30+
31+
}
32+
}

‎internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
(builtins.fetchClosure{
2727
fromStore = "{{ $.BinaryCacheStore }}";
2828
fromPath = "{{ .PathInBinaryStore }}";
29+
toPath = "{{ .PathInLocalStore }}";
2930
})
3031
{{- end }}
3132
{{- end }}

0 commit comments

Comments
 (0)