install packages via nix build

Author: savil

internal/devbox/packages.go

@@ -248,10 +248,23 @@ const (
 // The `mode` is used for:
 // 1. Skipping certain operations that may not apply.
 // 2. User messaging to explain what operations are happening, because this function may take time to execute.
-func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) error {
+//
+// nolint:revive to turn off linter complaining about function complexity
+func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) error { //nolint:revive
 	defer trace.StartRegion(ctx, "devboxEnsureStateIsUpToDate").End()
 	defer debug.FunctionTimer().End()
 
+	if mode != ensure && !d.IsEnvEnabled() {
+		// if mode is install/uninstall/update and we are not in a devbox environment,
+		// then we skip some operations below for speed.
+		// Remove the local.lock file so that we re-compute the project state when
+		// we are in the devbox environment again.
+		fmt.Printf("mode is not ensure and env is not enabled, so will remove local.lock on exit\n")
+
+		defer func() { _ = d.lockfile.RemoveLocal() }()
+		// return nil
+	}
+
 	// if mode is install or uninstall, then we need to update the nix-profile
 	// and lockfile, so we must continue below.
 	upToDate, err := d.lockfile.IsUpToDateAndInstalled()
@@ -262,10 +275,12 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er
 	if mode == ensure {
 		// if mode is ensure and we are up to date, then we can skip the rest
 		if upToDate {
+			fmt.Printf("state is up to date. Returning early\n")
 			return nil
 		}
 		fmt.Fprintln(d.stderr, "Ensuring packages are installed.")
 	}
+	fmt.Printf("state is not up to date. Continuing...\n")
 
 	// Validate packages. Must be run up-front and definitely prior to computeEnv
 	// and syncNixProfile below that will evaluate the flake and may give
@@ -285,6 +300,9 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er
 		return err
 	}
 
+	fmt.Printf("mode is %s\n", mode)
+	fmt.Printf("env is enabled: %v\n", d.IsEnvEnabled())
+
 	if err := shellgen.GenerateForPrintEnv(ctx, d); err != nil {
 		return err
 	}
@@ -293,24 +311,37 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er
 		return err
 	}
 
-	// Always re-compute print-dev-env to ensure all packages are installed, and
-	// the correct set of packages are reflected in the nix-profile below.
-	env, err := d.computeEnv(ctx, false /*usePrintDevEnvCache*/)
-	if err != nil {
-		return err
-	}
+	// The steps contained in this if-block of computeEnv and syncNixProfile are a tad
+	// slow. So, we only do it if we are in a devbox environment, or if mode is ensure.
+	if mode == ensure || d.IsEnvEnabled() {
+		// Re-compute print-dev-env to ensure all packages are installed, and
+		// the correct set of packages are reflected in the nix-profile below.
+		env, err := d.computeEnv(ctx, false /*usePrintDevEnvCache*/)
+		if err != nil {
+			return err
+		}
 
-	// Ensure the nix profile has the packages from the flake.
-	buildInputs := []string{}
-	if env["buildInputs"] != "" {
-		// env["buildInputs"] can be empty string if there are no packages in the project
-		// if buildInputs is empty, then we don't want wantStorePaths to be an array with a single "" entry
-		buildInputs = strings.Split(env["buildInputs"], " ")
-	}
-	if err := d.syncNixProfile(ctx, buildInputs); err != nil {
-		return err
+		// Ensure the nix profile has the packages from the flake.
+		buildInputs := []string{}
+		if env["buildInputs"] != "" {
+			// env["buildInputs"] can be empty string if there are no packages in the project
+			// if buildInputs is empty, then we don't want wantStorePaths to be an array with a single "" entry
+			buildInputs = strings.Split(env["buildInputs"], " ")
+		}
+		if err := d.syncNixProfile(ctx, buildInputs); err != nil {
+			return err
+		}
+
+	} else if mode == install || mode == update {
+		// Else: if we are not in a devbox environment, and we are installing or updating
+		// then we must ensure the new nix packages are in the nix store. This way, the
+		// next time we enter a devbox environment, we will have the packages available locally.
+		if err := d.installNixPackagesToStore(ctx); err != nil {
+			return err
+		}
 	}
 
+	fmt.Printf("calling lockfile.tidy\n")
 	// Ensure we clean out packages that are no longer needed.
 	d.lockfile.Tidy()
 
@@ -395,6 +426,33 @@ func (d *Devbox) InstallRunXPackages(ctx context.Context) error {
 	return nil
 }
 
+// installNixPackagesToStore will install all the packages in the nix store, if
+// mode is install or update, and we're not in a devbox environment.
+// This is done by running `nix build` on the flake
+func (d *Devbox) installNixPackagesToStore(ctx context.Context) error {
+	packages, err := d.AllInstallablePackages()
+	if err != nil {
+		return err
+	}
+	packages = lo.Filter(packages, devpkg.IsNix) // Remove non-nix packages from the list
+
+	installables := []string{}
+	for _, pkg := range packages {
+		i, err := pkg.Installable()
+		if err != nil {
+			return err
+		}
+		installables = append(installables, i)
+	}
+
+	if len(installables) == 0 {
+		return nil
+	}
+
+	// --no-link to avoid generating the result objects
+	return nix.Build(ctx, []string{"--no-link"}, installables...)
+}
+
 // validatePackagesToBeInstalled will ensure that packages are available to be installed
 // in the user's current system.
 func (d *Devbox) validatePackagesToBeInstalled(ctx context.Context) error {

internal/lock/local.go

@@ -5,7 +5,9 @@ package lock
 
 import (
 	"errors"
+	"fmt"
 	"io/fs"
+	"os"
 	"path/filepath"
 
 	"go.jetpack.io/devbox/internal/build"
@@ -49,6 +51,8 @@ func isLocalUpToDate(project devboxProject) (bool, error) {
 }
 
 func updateLocal(project devboxProject) error {
+	fmt.Printf("updating local.lock\n")
+	// debug.PrintStack()
 	l, err := readLocal(project)
 	if err != nil {
 		return err
@@ -74,6 +78,11 @@ func readLocal(project devboxProject) (*localLockFile, error) {
 	return lockFile, nil
 }
 
+func removeLocal(project devboxProject) error {
+	// RemoveAll to avoid error in case file does not exist.
+	return os.RemoveAll(localLockFilePath(project))
+}
+
 func forProject(project devboxProject) (*localLockFile, error) {
 	configHash, err := project.ConfigHash()
 	if err != nil {

internal/lock/lockfile.go

@@ -55,6 +55,7 @@ func (f *File) Add(pkgs ...string) error {
 			return err
 		}
 	}
+	fmt.Printf("calling lockfile.Add with pkgs: %v\n", pkgs)
 	return f.Save()
 }
 
@@ -176,6 +177,11 @@ func (f *File) isDirty() (bool, error) {
 	return currentHash != filesystemHash, nil
 }
 
+// RemoveLocal removes the local.lock file
+func (f *File) RemoveLocal() error {
+	return removeLocal(f.devboxProject)
+}
+
 func lockFilePath(project devboxProject) string {
 	return filepath.Join(project.ProjectDir(), "devbox.lock")
 }

internal/nix/build.go

@@ -0,0 +1,29 @@
+package nix
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+
+	"github.com/pkg/errors"
+)
+
+func Build(ctx context.Context, flags []string, installables ...string) error {
+	// --impure is required for allowUnfreeEnv to work.
+	cmd := commandContext(ctx, "build", "--impure")
+	cmd.Args = append(cmd.Args, flags...)
+	cmd.Args = append(cmd.Args, installables...)
+	// We need to allow Unfree packages to be installed. We choose to not also add os.Environ() to the environment
+	// to keep the command as pure as possible, even though we must pass --impure to nix build.
+	cmd.Env = allowUnfreeEnv([]string{}) // allowUnfreeEnv(os.Environ())
+
+	out, err := cmd.Output()
+	if err != nil {
+		if exitErr := (&exec.ExitError{}); errors.As(err, &exitErr) {
+			fmt.Printf("Exit code: %d, output: %s\n", exitErr.ExitCode(), exitErr.Stderr)
+		}
+		return err
+	}
+	fmt.Printf("Ran nix build, output: %s\n", out)
+	return nil
+}

testscripts/add/global_add.test.txt

@@ -4,7 +4,7 @@
 ! exec vim --version
 exec devbox global add ripgrep vim
 
-exec devbox global shellenv
+exec devbox global shellenv --recompute
 source.path
 exec rg --version
 exec vim --version