[remove nixpkgs] part 2: generate flake.nix using builtins.fetchClosure (#1209)

Author: savil

internal/lock/lockfile.go

@@ -4,12 +4,12 @@
 package lock
 
 import (
-	"errors"
 	"fmt"
 	"io/fs"
 	"path/filepath"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 

internal/nix/input.go

@@ -13,8 +13,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/samber/lo"
-
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/lock"
@@ -73,6 +73,7 @@ func PackageFromString(raw string, locker lock.Locker) *Package {
 		}
 		pkgURL, _ = url.Parse(normalizedURL)
 	}
+
 	return &Package{*pkgURL, locker, raw, ""}
 }
 
@@ -407,6 +408,62 @@ func (p *Package) hashFromNixPkgsURL() string {
 	return HashFromNixPkgsURL(p.URLForFlakeInput())
 }
 
+// BinaryCacheStore is the store from which to fetch this package's binaries.
+// It is used as FromStore in builtins.fetchClosure.
+const BinaryCacheStore = "https://cache.nixos.org"
+
+func (p *Package) IsInBinaryStore() (bool, error) {
+	if !p.isVersioned() {
+		return false, nil
+	}
+
+	entry, err := p.lockfile.Resolve(p.Raw)
+	if err != nil {
+		return false, err
+	}
+
+	userSystem, err := System()
+	if err != nil {
+		return false, err
+	}
+
+	if entry.Systems == nil {
+		return false, nil
+	}
+
+	// Check if the user's system's info is present in the lockfile
+	_, ok := entry.Systems[userSystem]
+	return ok, nil
+}
+
+// PathInBinaryStore is the key in the BinaryCacheStore for this package
+// This is used as FromPath in builtins.fetchClosure
+func (p *Package) PathInBinaryStore() (string, error) {
+	if isInStore, err := p.IsInBinaryStore(); err != nil {
+		return "", err
+	} else if !isInStore {
+		return "", errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw)
+	}
+
+	entry, err := p.lockfile.Resolve(p.Raw)
+	if err != nil {
+		return "", err
+	}
+
+	userSystem, err := System()
+	if err != nil {
+		return "", err
+	}
+
+	sysInfo := entry.Systems[userSystem]
+	storeDirParts := []string{sysInfo.FromHash, sysInfo.StoreName}
+	if sysInfo.StoreVersion != "" {
+		storeDirParts = append(storeDirParts, sysInfo.StoreVersion)
+	}
+	storeDir := strings.Join(storeDirParts, "-")
+	return filepath.Join("/nix/store", storeDir), nil
+}
+
 // IsGithubNixpkgsURL returns true if the package is a flake of the form:
 // github:NixOS/nixpkgs/...
 //

internal/nix/nix.go

@@ -11,8 +11,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime/trace"
+	"strings"
 
 	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 
 	"go.jetpack.io/devbox/internal/debug"
 )
@@ -64,6 +66,9 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 			args.FlakesFilePath,
 		)
 		cmd.Args = append(cmd.Args, ExperimentalFlags()...)
+		if featureflag.RemoveNixpkgs.Enabled() {
+			cmd.Args = append(cmd.Args, "--impure")
+		}
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
 		data, err = cmd.Output()
@@ -102,15 +107,27 @@ func FlakeNixpkgs(commit string) string {
 }
 
 func ExperimentalFlags() []string {
+	options := []string{"nix-command", "flakes"}
+	if featureflag.RemoveNixpkgs.Enabled() {
+		options = append(options, "fetch-closure")
+	}
 	return []string{
 		"--extra-experimental-features", "ca-derivations",
-		"--option", "experimental-features", "nix-command flakes",
+		"--option", "experimental-features", strings.Join(options, " "),
 	}
 }
 
 var cachedSystem string
 
 func System() (string, error) {
+	// For Savil to debug "remove nixpkgs" feature. The search api lacks x86-darwin info.
+	// So, I need to fake that I am x86-linux and inspect the output in generated devbox.lock
+	// and flake.nix files.
+	override := os.Getenv("__DEVBOX_NIX_SYSTEM")
+	if override != "" {
+		return override, nil
+	}
+
 	if cachedSystem == "" {
 		cmd := exec.Command(
 			"nix", "eval", "--impure", "--raw", "--expr", "builtins.currentSystem",

internal/shellgen/flake_input.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/samber/lo"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/goutil"
 	"go.jetpack.io/devbox/internal/nix"
 )
@@ -64,24 +65,30 @@ func (f *flakeInput) BuildInputs() []string {
 // i.e. have a commit hash and always resolve to the same package/version.
 // Note: inputs returned by this function include plugin packages. (php only for now)
 // It's not entirely clear we always want to add plugin packages to the top level
-func flakeInputs(ctx context.Context, devbox devboxer) ([]*flakeInput, error) {
+func flakeInputs(ctx context.Context, packages []*nix.Package) ([]*flakeInput, error) {
 	defer trace.StartRegion(ctx, "flakeInputs").End()
 
 	// Use the verbose name flakeInputs to distinguish from `inputs`
 	// which refer to `nix.Input` in most of the codebase.
 	flakeInputs := map[string]*flakeInput{}
 
-	userInputs := devbox.PackagesAsInputs()
-	pluginInputs, err := devbox.PluginManager().PluginInputs(userInputs)
-	if err != nil {
-		return nil, err
-	}
+	packages = lo.Filter(packages, func(item *nix.Package, _ int) bool {
+		// Include packages (like local or remote flakes) that cannot be
+		// fetched from a Binary Cache Store.
+		if !featureflag.RemoveNixpkgs.Enabled() {
+			return true
+		}
+
+		inStore, err := item.IsInBinaryStore()
+		if err != nil {
+			// Ignore this error for now. TODO savil: return error?
+			return true
+		}
+		return !inStore
+	})
 
 	order := []string{}
-	// We prioritize plugin packages so that the php plugin works. Not sure
-	// if this is behavior we want for user plugins. We may need to add an optional
-	// priority field to the config.
-	for _, input := range append(pluginInputs, userInputs...) {
+	for _, input := range packages {
 		AttributePath, err := input.FullPackageAttributePath()
 		if err != nil {
 			return nil, err

internal/shellgen/flake_plan.go

@@ -3,13 +3,18 @@ package shellgen
 import (
 	"context"
 	"runtime/trace"
+
+	"go.jetpack.io/devbox/internal/nix"
 )
 
 // flakePlan contains the data to populate the top level flake.nix file
 // that builds the devbox environment
 type flakePlan struct {
-	NixpkgsInfo *NixpkgsInfo
-	FlakeInputs []*flakeInput
+	BinaryCacheStore string
+	NixpkgsInfo      *NixpkgsInfo
+	FlakeInputs      []*flakeInput
+	Packages         []*nix.Package
+	System           string
 }
 
 func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
@@ -35,9 +40,17 @@ func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
 		}
 	}
 
-	shellPlan := &flakePlan{}
-	var err error
-	shellPlan.FlakeInputs, err = flakeInputs(ctx, devbox)
+	userPackages := devbox.PackagesAsInputs()
+	pluginPackages, err := devbox.PluginManager().PluginInputs(userPackages)
+	if err != nil {
+		return nil, err
+	}
+	// We prioritize plugin packages so that the php plugin works. Not sure
+	// if this is behavior we want for user plugins. We may need to add an optional
+	// priority field to the config.
+	packages := append(pluginPackages, userPackages...)
+
+	flakeInputs, err := flakeInputs(ctx, packages)
 	if err != nil {
 		return nil, err
 	}
@@ -46,14 +59,23 @@ func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
 
 	// This is an optimization. Try to reuse the nixpkgs info from the flake
 	// inputs to avoid introducing a new one.
-	for _, input := range shellPlan.FlakeInputs {
+	for _, input := range flakeInputs {
 		if input.IsNixpkgs() {
 			nixpkgsInfo = getNixpkgsInfo(input.HashFromNixPkgsURL())
 			break
 		}
 	}
 
-	shellPlan.NixpkgsInfo = nixpkgsInfo
+	system, err := nix.System()
+	if err != nil {
+		return nil, err
+	}
 
-	return shellPlan, nil
+	return &flakePlan{
+		BinaryCacheStore: nix.BinaryCacheStore,
+		FlakeInputs:      flakeInputs,
+		NixpkgsInfo:      nixpkgsInfo,
+		Packages:         packages,
+		System:           system,
+	}, nil
 }

internal/shellgen/generate.go

@@ -17,15 +17,14 @@ import (
 	"text/template"
 
 	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/debug"
 )
 
 //go:embed tmpl/*
 var tmplFS embed.FS
 
-var shellFiles = []string{"shell.nix"}
-
 // GenerateForPrintEnv will create all the files necessary for processing
 // devbox.PrintEnv, which is the core function from which devbox shell/run/direnv
 // functionality is derived.
@@ -39,15 +38,14 @@ func GenerateForPrintEnv(ctx context.Context, devbox devboxer) error {
 
 	outPath := genPath(devbox)
 
-	for _, file := range shellFiles {
-		err := writeFromTemplate(outPath, plan, file)
-		if err != nil {
-			return errors.WithStack(err)
-		}
+	// Preserving shell.nix to avoid breaking old-style .envrc users
+	err = writeFromTemplate(outPath, plan, "shell.nix", "shell.nix")
+	if err != nil {
+		return errors.WithStack(err)
 	}
 
 	// Gitignore file is added to the .devbox directory
-	err = writeFromTemplate(filepath.Join(devbox.ProjectDir(), ".devbox"), plan, ".gitignore")
+	err = writeFromTemplate(filepath.Join(devbox.ProjectDir(), ".devbox"), plan, ".gitignore", ".gitignore")
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -69,7 +67,7 @@ var (
 	tmplOldBuf = bytes.NewBuffer(make([]byte, 0, 4096))
 )
 
-func writeFromTemplate(path string, plan any, tmplName string) error {
+func writeFromTemplate(path string, plan any, tmplName string, generatedName string) error {
 	tmplKey := tmplName + ".tmpl"
 	tmpl := tmplCache[tmplKey]
 	if tmpl == nil {
@@ -93,7 +91,7 @@ func writeFromTemplate(path string, plan any, tmplName string) error {
 	// changed. Blindly overwriting the file could invalidate Nix's cache
 	// every time, slowing down evaluation considerably.
 	var (
-		outPath = filepath.Join(path, tmplName)
+		outPath = filepath.Join(path, generatedName)
 		flag    = os.O_RDWR | os.O_CREATE
 		perm    = fs.FileMode(0644)
 	)
@@ -150,7 +148,11 @@ var templateFuncs = template.FuncMap{
 
 func makeFlakeFile(d devboxer, outPath string, plan *flakePlan) error {
 	flakeDir := FlakePath(d)
-	err := writeFromTemplate(flakeDir, plan, "flake.nix")
+	templateName := "flake.nix"
+	if featureflag.RemoveNixpkgs.Enabled() {
+		templateName = "flake_remove_nixpkgs.nix"
+	}
+	err := writeFromTemplate(flakeDir, plan, templateName, "flake.nix")
 	if err != nil {
 		return errors.WithStack(err)
 	}

internal/shellgen/generate_test.go

@@ -19,14 +19,14 @@ var update = flag.Bool("update", false, "update the golden files with the test r
 func TestWriteFromTemplate(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "makeme")
 	outPath := filepath.Join(dir, "flake.nix")
-	err := writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix")
+	err := writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix", "flake.nix")
 	if err != nil {
 		t.Fatal("got error writing flake template:", err)
 	}
 	cmpGoldenFile(t, outPath, "testdata/flake.nix.golden")
 
 	t.Run("WriteUnmodified", func(t *testing.T) {
-		err = writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix")
+		err = writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix", "flake.nix")
 		if err != nil {
 			t.Fatal("got error writing flake template:", err)
 		}
@@ -39,7 +39,7 @@ func TestWriteFromTemplate(t *testing.T) {
 			}
 			FlakeInputs []flakeInput
 		}{}
-		err = writeFromTemplate(dir, emptyPlan, "flake.nix")
+		err = writeFromTemplate(dir, emptyPlan, "flake.nix", "flake.nix")
 		if err != nil {
 			t.Fatal("got error writing flake template:", err)
 		}

internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

@@ -0,0 +1,40 @@
+{
+   description = "A devbox shell";
+
+   inputs = {
+     nixpkgs.url = "{{ .NixpkgsInfo.URL }}";
+     {{- range .FlakeInputs }}
+     {{.Name}}.url = "{{.URLWithCaching}}";
+     {{- end }}
+   };
+
+   outputs = {
+     self,
+     nixpkgs,
+     {{- range .FlakeInputs }}
+     {{.Name}},
+     {{- end }}
+   }:
+      let
+        pkgs = nixpkgs.legacyPackages.{{ .System }};
+      in
+      {
+        devShells.{{ .System }}.default = pkgs.mkShell {
+          buildInputs = [
+            {{- range .Packages }}
+            {{- if .IsInBinaryStore }}
+            (builtins.fetchClosure{
+              fromStore = "{{ $.BinaryCacheStore }}";
+              fromPath = "{{ .PathInBinaryStore }}";
+            })
+            {{- end }}
+            {{- end }}
+            {{- range .FlakeInputs }}
+            {{- range .BuildInputs }}
+            {{.}}
+            {{- end }}
+            {{- end }}
+          ];
+        };
+      };
+ }