ensure packages are installable, and cleanup, enable php testscript

Author: savil

.github/workflows/cli-tests.yaml

@@ -129,9 +129,8 @@ jobs:
         run-project-tests: ["project-tests", "project-tests-off"]
         # Run tests on:
         # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0)
-        # 2. nix version 2.17.0 which introduces a new code path that minimizes nixpkgs downloads.
-        # 3. latest nix version
-        nix-version: ["2.12.0", "2.17.0", "2.19.2"]
+        # 2. latest nix version, must be > 2.17.0 which introduces a new code path that minimizes nixpkgs downloads.
+        nix-version: ["2.12.0", "2.19.2"]
         exclude:
           - is-main: "not-main"
             os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}"

internal/devbox/nixprofile.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"slices"
 	"strings"
 
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/nix/nixprofile"
 )
 
+// syncFlakeToProfile ensures the buildInputs from the flake's devShell are
+// installed in the nix profile.
 func (d *Devbox) syncFlakeToProfile(ctx context.Context) error {
 	profilePath, err := d.profilePath()
 	if err != nil {
@@ -73,6 +74,7 @@ func (d *Devbox) syncFlakeToProfile(ctx context.Context) error {
 				Installable:       addPath,
 				// Install in offline mode for speed. We know we should have all the files
 				// locally in /nix/store since we have run `nix print-dev-env` prior to this.
+				// Also avoids some "substituter not found for store-path" errors.
 				Offline:     true,
 				PackageName: storePath.Name,
 				ProfilePath: profilePath,
@@ -86,30 +88,24 @@ func (d *Devbox) syncFlakeToProfile(ctx context.Context) error {
 }
 
 func diffStorePaths(got, want []string) (add, remove []string) {
-	slices.Sort(got)
-	slices.Sort(want)
+	gotSet := map[string]bool{}
+	for _, g := range got {
+		gotSet[g] = true
+	}
+	wantSet := map[string]bool{}
+	for _, w := range want {
+		wantSet[w] = true
+	}
 
-	var gotIdx, wantIdx int
-	for {
-		if gotIdx >= len(got) {
-			add = append(add, want[wantIdx:]...)
-			break
-		}
-		if wantIdx >= len(want) {
-			remove = append(remove, got[gotIdx:]...)
-			break
+	for _, g := range got {
+		if _, ok := wantSet[g]; !ok {
+			remove = append(remove, g)
 		}
+	}
 
-		switch {
-		case got[gotIdx] == want[wantIdx]:
-			gotIdx++
-			wantIdx++
-		case got[gotIdx] < want[wantIdx]:
-			remove = append(remove, got[gotIdx])
-			gotIdx++
-		case got[gotIdx] > want[wantIdx]:
-			add = append(add, want[wantIdx])
-			wantIdx++
+	for _, w := range want {
+		if _, ok := gotSet[w]; !ok {
+			add = append(add, w)
 		}
 	}
 	return add, remove

internal/devbox/packages.go

@@ -266,6 +266,13 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er
 		fmt.Fprintln(d.stderr, "Ensuring packages are installed.")
 	}
 
+	// Validate packages. Must be run up-front and definitely prior to computeEnv
+	// and syncFlakeToProfile below that will evaluate the flake and may give
+	// inscrutable errors if the package is uninstallable.
+	if err := d.validatePackages(); err != nil {
+		return err
+	}
+
 	// Create plugin directories first because packages might need them
 	for _, pkg := range d.InstallablePackages() {
 		if err := d.PluginManager().Create(pkg); err != nil {
@@ -285,13 +292,15 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er
 		return err
 	}
 
-	// Use the printDevEnvCache if we are adding or removing or updating any package,
-	// AND we are not in the shellenv-enabled environment of the current devbox-project.
-	usePrintDevEnvCache := mode != ensure && !d.IsEnvEnabled()
-	if _, err := d.computeEnv(ctx, usePrintDevEnvCache); err != nil {
+	// Skip the print-dev-env's cache if we are in a devbox-environment. If not,
+	// skip the cache if we're either installing packages or ensuring
+	// the project state is up-to-date.
+	skipPrintDevEnvCache := d.IsEnvEnabled() || (mode == ensure || mode == install)
+	if _, err := d.computeEnv(ctx, !skipPrintDevEnvCache /*usePrintDevEnvCache*/); err != nil {
 		return err
 	}
 
+	// Ensure the nix profile has the packages from the flake.
 	if err := d.syncFlakeToProfile(ctx); err != nil {
 		return err
 	}
@@ -379,3 +388,37 @@ func (d *Devbox) InstallRunXPackages(ctx context.Context) error {
 	}
 	return nil
 }
+
+// validatePackages will ensure that packages are available to be installed
+// in the user's current system.
+func (d *Devbox) validatePackages() error {
+	for _, pkg := range d.InstallablePackages() {
+
+		inCache, err := pkg.IsInBinaryCache()
+		if err != nil {
+			return err
+		}
+
+		if !inCache && nix.IsGithubNixpkgsURL(pkg.URLForFlakeInput()) {
+			if err := nix.EnsureNixpkgsPrefetched(d.stderr, pkg.HashFromNixPkgsURL()); err != nil {
+				return err
+			}
+			if exists, err := pkg.ValidateInstallsOnSystem(); err != nil {
+				return err
+			} else if !exists {
+				platform := nix.System()
+				return usererr.New(
+					"package %s cannot be installed on your platform %s.\n"+
+						"If you know this package is incompatible with %[2]s, then "+
+						"you could run `devbox add %[1]s --exclude-platform %[2]s` and re-try.\n"+
+						"If you think this package should be compatible with %[2]s, then "+
+						"it's possible this particular version is not available yet from the nix registry. "+
+						"You could try `devbox add` with a different version for this package.\n",
+					pkg.Raw,
+					platform,
+				)
+			}
+		}
+	}
+	return nil
+}

internal/nix/nixprofile/profile.go

@@ -15,7 +15,6 @@ import (
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
-	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/nix"
@@ -207,35 +206,8 @@ type ProfileInstallPackageArgs struct {
 // ProfileInstallPackage installs a Devbox package into a profile.
 // TODO drop the Profile prefix from this name.
 func ProfileInstallPackage(ctx context.Context, args *ProfileInstallPackageArgs) error {
-	input := devpkg.PackageFromStringWithDefaults(args.Package, args.Lockfile)
-
-	inCache, err := input.IsInBinaryCache()
-	if err != nil {
-		return err
-	}
-
-	if !inCache && nix.IsGithubNixpkgsURL(input.URLForFlakeInput()) {
-		if err := nix.EnsureNixpkgsPrefetched(args.Writer, input.HashFromNixPkgsURL()); err != nil {
-			return err
-		}
-		if exists, err := input.ValidateInstallsOnSystem(); err != nil {
-			return err
-		} else if !exists {
-			platform := nix.System()
-			return usererr.New(
-				"package %s cannot be installed on your platform %s.\n"+
-					"If you know this package is incompatible with %[2]s, then "+
-					"you could run `devbox add %[1]s --exclude-platform %[2]s` and re-try.\n"+
-					"If you think this package should be compatible with %[2]s, then "+
-					"it's possible this particular version is not available yet from the nix registry. "+
-					"You could try `devbox add` with a different version for this package.\n",
-				input.Raw,
-				platform,
-			)
-		}
-	}
-
-	installable, err := input.Installable()
+	pkg := devpkg.PackageFromStringWithDefaults(args.Package, args.Lockfile)
+	installable, err := pkg.Installable()
 	if err != nil {
 		return err
 	}
@@ -244,6 +216,7 @@ func ProfileInstallPackage(ctx context.Context, args *ProfileInstallPackageArgs)
 		Installable:       installable,
 		PackageName:       args.Package,
 		ProfilePath:       args.ProfilePath,
+		Writer:            args.Writer,
 	})
 }
 

testscripts/languages/php.test.txt

@@ -1,23 +1,19 @@
 exec devbox run php index.php
 stdout 'done\n'
 
-# temporarily disable rm test until we fix. It requires ensuring that package 
-# in profile matches the flake we are installing.
-# For non-flakes, this involves comparing versions. For flakes we need to compare 
-# the content (or hash)
-
-# exec devbox rm php81Extensions.ds
-# exec devbox run php index.php
-# stdout 'ds extension is not enabled'
+exec devbox rm php81Extensions.ds
+exec devbox run php index.php
+stdout 'ds extension is not enabled'
 
-# exec devbox add php81Extensions.ds
-# exec devbox run php index.php
-# stdout 'done\n'
+exec devbox add php81Extensions.ds
+exec devbox run php index.php
+stdout 'done\n'
 
 -- devbox.json --
 {
   "packages": [
     "php81@latest",
+    "php81Packages.composer@latest",
     "php81Extensions.ds@latest"
   ]
 }