[remove nixpkgs] part 2: generate flake.nix using builtins.fetchClosure

Author: savil

internal/lock/interfaces.go

@@ -18,4 +18,5 @@ type Locker interface {
 	resolver
 	LegacyNixpkgsPath(string) string
 	ProjectDir() string
+	SystemInfo(pkg string) (*SystemInfo, error)
 }

internal/lock/lockfile.go

@@ -4,12 +4,12 @@
 package lock
 
 import (
-	"errors"
 	"fmt"
 	"io/fs"
 	"path/filepath"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 
@@ -129,6 +129,33 @@ func (l *File) Resolve(pkg string) (*Package, error) {
 	return l.Packages[pkg], nil
 }
 
+func (l *File) SystemInfo(pkgVersionedName string) (*SystemInfo, error) {
+	if !featureflag.RemoveNixpkgs.Enabled() {
+		return nil, nil
+	}
+
+	pkg, ok := l.Packages[pkgVersionedName]
+	if !ok {
+		return nil, errors.Errorf("package %s not found in lockfile", pkgVersionedName)
+	}
+
+	if sysInfo, ok := pkg.Systems[l.system]; ok {
+		// Found it! We are done.
+		return sysInfo, nil
+	}
+
+	// If the systemInfo is missing, then resolve the package to get it from the search api.
+	var err error
+	pkg, err = l.Resolve(pkgVersionedName)
+	if err != nil {
+		return nil, err
+	}
+	if sysInfo, ok := pkg.Systems[l.system]; ok {
+		return sysInfo, nil
+	}
+	return nil, errors.Errorf("Unable to get system %s info for package %s", l.system, pkgVersionedName)
+}
+
 func (l *File) ForceResolve(pkg string) (*Package, error) {
 	delete(l.Packages, pkg)
 	return l.Resolve(pkg)

internal/nix/input.go

@@ -14,9 +14,9 @@ import (
 	"strings"
 
 	"github.com/samber/lo"
-
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cuecfg"
+	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/lock"
 )
 
@@ -42,6 +42,9 @@ type Package struct {
 	Raw string
 
 	normalizedPackageAttributePathCache string // memoized value from normalizedPackageAttributePath()
+
+	// sysInfo contains system-specific information about this package in the nix store
+	sysInfo *lock.SystemInfo
 }
 
 // PackageFromStrings constructs Package from the list of package names provided.
@@ -73,7 +76,26 @@ func PackageFromString(raw string, locker lock.Locker) *Package {
 		}
 		pkgURL, _ = url.Parse(normalizedURL)
 	}
-	return &Package{*pkgURL, locker, raw, ""}
+
+	pkg := &Package{
+		URL:                                 *pkgURL,
+		lockfile:                            locker,
+		Raw:                                 raw,
+		normalizedPackageAttributePathCache: "",
+		sysInfo:                             nil,
+	}
+
+	if pkg.isVersioned() {
+		sysInfo, err := locker.SystemInfo(raw)
+		if err != nil {
+			// ignoring for coding convenience. TODO savil. handle error
+			debug.Log("ERROR: failed to get locker.SystemInfo for pkg %s. Error is %s", raw, err)
+		} else if sysInfo != nil {
+			pkg.sysInfo = sysInfo
+		}
+	}
+
+	return pkg
 }
 
 // PackageFromProfileItem constructs a package using the the unlocked reference
@@ -407,6 +429,10 @@ func (p *Package) hashFromNixPkgsURL() string {
 	return HashFromNixPkgsURL(p.URLForFlakeInput())
 }
 
+func (p *Package) SystemInfo() *lock.SystemInfo {
+	return p.sysInfo
+}
+
 // IsGithubNixpkgsURL returns true if the package is a flake of the form:
 // github:NixOS/nixpkgs/...
 //

internal/nix/input_test.go

@@ -131,6 +131,10 @@ func (l *lockfile) Resolve(pkg string) (*lock.Package, error) {
 	}
 }
 
+func (l *lockfile) SystemInfo(pkg string) (*lock.SystemInfo, error) {
+	return nil, nil
+}
+
 func testInputFromString(s, projectDir string) *testInput {
 	return lo.ToPtr(testInput{Package: *PackageFromString(s, &lockfile{projectDir})})
 }

internal/nix/nix.go

@@ -11,8 +11,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime/trace"
+	"strings"
 
 	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 
 	"go.jetpack.io/devbox/internal/debug"
 )
@@ -64,6 +66,9 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 			args.FlakesFilePath,
 		)
 		cmd.Args = append(cmd.Args, ExperimentalFlags()...)
+		if featureflag.RemoveNixpkgs.Enabled() {
+			cmd.Args = append(cmd.Args, "--impure")
+		}
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
 		data, err = cmd.Output()
@@ -102,15 +107,27 @@ func FlakeNixpkgs(commit string) string {
 }
 
 func ExperimentalFlags() []string {
+	options := []string{"nix-command", "flakes"}
+	if featureflag.RemoveNixpkgs.Enabled() {
+		options = append(options, "fetch-closure")
+	}
 	return []string{
 		"--extra-experimental-features", "ca-derivations",
-		"--option", "experimental-features", "nix-command flakes",
+		"--option", "experimental-features", strings.Join(options, " "),
 	}
 }
 
 var cachedSystem string
 
 func System() (string, error) {
+	// For Savil to debug "remove nixpkgs" feature. The search api lacks x86-darwin info.
+	// So, I need to fake that I am x86-linux and inspect the output in generated devbox.lock
+	// and flake.nix files.
+	override := os.Getenv("__DEVBOX_NIX_SYSTEM")
+	if override != "" {
+		return override, nil
+	}
+
 	if cachedSystem == "" {
 		cmd := exec.Command(
 			"nix", "eval", "--impure", "--raw", "--expr", "builtins.currentSystem",

internal/shellgen/flake_input.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/samber/lo"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/goutil"
 	"go.jetpack.io/devbox/internal/nix"
 )
@@ -77,11 +78,18 @@ func flakeInputs(ctx context.Context, devbox devboxer) ([]*flakeInput, error) {
 		return nil, err
 	}
 
-	order := []string{}
 	// We prioritize plugin packages so that the php plugin works. Not sure
 	// if this is behavior we want for user plugins. We may need to add an optional
 	// priority field to the config.
-	for _, input := range append(pluginInputs, userInputs...) {
+	allInputs := append(pluginInputs, userInputs...)
+	allInputs = lo.Filter(allInputs, func(item *nix.Package, _ int) bool {
+		// This is temporary, so that we can support the existing flake.nix.tmpl
+		// as well as the new flake_remove_nixpkgs.nix.tmpl.
+		return !featureflag.RemoveNixpkgs.Enabled() || item.SystemInfo() == nil
+	})
+
+	order := []string{}
+	for _, input := range allInputs {
 		AttributePath, err := input.FullPackageAttributePath()
 		if err != nil {
 			return nil, err

internal/shellgen/flake_package.go

@@ -0,0 +1,66 @@
+package shellgen
+
+import (
+	"path/filepath"
+	"strings"
+
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
+	"go.jetpack.io/devbox/internal/nix"
+)
+
+type flakePackage struct {
+	Name      string
+	FromStore string
+	FromPath  string
+}
+
+func newFlakePackage(pkg *nix.Package) (*flakePackage, error) {
+	// nixosCacheURL is where we fetch package binaries from
+	const nixosCacheURL = "https://cache.nixos.org"
+
+	// flakePackages only support versioned packages that have a system info
+	sysInfo := pkg.SystemInfo()
+	if sysInfo == nil {
+		return nil, nil
+	}
+
+	attributePath, err := pkg.PackageAttributePath()
+	if err != nil {
+		return nil, err
+	}
+
+	storeDir := strings.Join([]string{sysInfo.FromHash, sysInfo.StoreName, sysInfo.StoreVersion}, "-")
+	return &flakePackage{
+		Name:      attributePath,
+		FromStore: nixosCacheURL,
+		FromPath:  filepath.Join("/nix/store", storeDir),
+		// TODO add ToPath:
+	}, nil
+}
+
+func flakePackages(devbox devboxer) ([]*flakePackage, error) {
+	if !featureflag.RemoveNixpkgs.Enabled() {
+		return nil, nil
+	}
+
+	userInputs := devbox.PackagesAsInputs()
+	pluginInputs, err := devbox.PluginManager().PluginInputs(userInputs)
+	if err != nil {
+		return nil, err
+	}
+	// As per flakeInputs function comments, we prioritize plugin packages
+	// so the php plugin works.
+	pkgs := append(pluginInputs, userInputs...)
+
+	result := []*flakePackage{}
+	for _, pkg := range pkgs {
+		flakePkg, err := newFlakePackage(pkg)
+		if err != nil {
+			return nil, err
+		}
+		if flakePkg != nil {
+			result = append(result, flakePkg)
+		}
+	}
+	return result, nil
+}

internal/shellgen/flake_plan.go

@@ -3,13 +3,17 @@ package shellgen
 import (
 	"context"
 	"runtime/trace"
+
+	"go.jetpack.io/devbox/internal/nix"
 )
 
 // flakePlan contains the data to populate the top level flake.nix file
 // that builds the devbox environment
 type flakePlan struct {
 	NixpkgsInfo *NixpkgsInfo
 	FlakeInputs []*flakeInput
+	Packages    []*flakePackage
+	System      string
 }
 
 func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
@@ -35,9 +39,13 @@ func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
 		}
 	}
 
-	shellPlan := &flakePlan{}
 	var err error
-	shellPlan.FlakeInputs, err = flakeInputs(ctx, devbox)
+	flakeInputs, err := flakeInputs(ctx, devbox)
+	if err != nil {
+		return nil, err
+	}
+
+	flakePackages, err := flakePackages(devbox)
 	if err != nil {
 		return nil, err
 	}
@@ -46,14 +54,22 @@ func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
 
 	// This is an optimization. Try to reuse the nixpkgs info from the flake
 	// inputs to avoid introducing a new one.
-	for _, input := range shellPlan.FlakeInputs {
+	for _, input := range flakeInputs {
 		if input.IsNixpkgs() {
 			nixpkgsInfo = getNixpkgsInfo(input.HashFromNixPkgsURL())
 			break
 		}
 	}
 
-	shellPlan.NixpkgsInfo = nixpkgsInfo
+	system, err := nix.System()
+	if err != nil {
+		return nil, err
+	}
 
-	return shellPlan, nil
+	return &flakePlan{
+		FlakeInputs: flakeInputs,
+		NixpkgsInfo: nixpkgsInfo,
+		Packages:    flakePackages,
+		System:      system,
+	}, nil
 }

internal/shellgen/generate.go

@@ -17,15 +17,14 @@ import (
 	"text/template"
 
 	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/debug"
 )
 
 //go:embed tmpl/*
 var tmplFS embed.FS
 
-var shellFiles = []string{"shell.nix"}
-
 // GenerateForPrintEnv will create all the files necessary for processing
 // devbox.PrintEnv, which is the core function from which devbox shell/run/direnv
 // functionality is derived.
@@ -39,15 +38,14 @@ func GenerateForPrintEnv(ctx context.Context, devbox devboxer) error {
 
 	outPath := genPath(devbox)
 
-	for _, file := range shellFiles {
-		err := writeFromTemplate(outPath, plan, file)
-		if err != nil {
-			return errors.WithStack(err)
-		}
+	// Preserving shell.nix to avoid breaking old-style .envrc users
+	err = writeFromTemplate(outPath, plan, "shell.nix", "shell.nix")
+	if err != nil {
+		return errors.WithStack(err)
 	}
 
 	// Gitignore file is added to the .devbox directory
-	err = writeFromTemplate(filepath.Join(devbox.ProjectDir(), ".devbox"), plan, ".gitignore")
+	err = writeFromTemplate(filepath.Join(devbox.ProjectDir(), ".devbox"), plan, ".gitignore", ".gitignore")
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -69,7 +67,7 @@ var (
 	tmplOldBuf = bytes.NewBuffer(make([]byte, 0, 4096))
 )
 
-func writeFromTemplate(path string, plan any, tmplName string) error {
+func writeFromTemplate(path string, plan any, tmplName string, generatedName string) error {
 	tmplKey := tmplName + ".tmpl"
 	tmpl := tmplCache[tmplKey]
 	if tmpl == nil {
@@ -93,7 +91,7 @@ func writeFromTemplate(path string, plan any, tmplName string) error {
 	// changed. Blindly overwriting the file could invalidate Nix's cache
 	// every time, slowing down evaluation considerably.
 	var (
-		outPath = filepath.Join(path, tmplName)
+		outPath = filepath.Join(path, generatedName)
 		flag    = os.O_RDWR | os.O_CREATE
 		perm    = fs.FileMode(0644)
 	)
@@ -150,7 +148,11 @@ var templateFuncs = template.FuncMap{
 
 func makeFlakeFile(d devboxer, outPath string, plan *flakePlan) error {
 	flakeDir := FlakePath(d)
-	err := writeFromTemplate(flakeDir, plan, "flake.nix")
+	templateName := "flake.nix"
+	if featureflag.RemoveNixpkgs.Enabled() {
+		templateName = "flake_remove_nixpkgs.nix"
+	}
+	err := writeFromTemplate(flakeDir, plan, templateName, "flake.nix")
 	if err != nil {
 		return errors.WithStack(err)
 	}