[allow-insecure] Allow insecure packages with --allow-insecure flag (#1265)

## Summary

This allows installing insecure packages using the `--allow-insecure`
flag:

`devbox add nodejs@16 --allow-insecure`

This saves the allow insecure state to lock file.

If user tries to do add/shell/run/install and there are insecure
pacakges that are not in marked in lock file, they will see an error
indicating they should use flag.

I used flag (instead of prompt) to limit the size of this already
massive PR. TODO (in follow up):

* When installing an insecure package, ask the user if they want to
allow it, update the devbox.json, and install it.

## How was it tested?

```bash
devbox add nodejs@16
devbox add nodejs@16 --allow-insecure
devbox run run_test
# edited lockfile to remove allow_insecure
devbox run run_test # error
devbox install # error
devbox shell # error
```

See examples/insecure

Author: mikeland73

devbox.json

@@ -22,4 +22,4 @@
   "nixpkgs": {
     "commit": "3364b5b117f65fe1ce65a3cdd5612a078a3b31e3"
   }
-}
+}

examples/insecure/devbox.json

@@ -0,0 +1,15 @@
+{
+  "packages": [
+    "nodejs@16"
+  ],
+  "shell": {
+    "init_hook": [
+      "echo 'Welcome to devbox!' > /dev/null"
+    ],
+    "scripts": {
+      "run_test": [
+        "node --version"
+      ]
+    }
+  }
+}

examples/insecure/devbox.lock

@@ -0,0 +1,12 @@
+{
+  "lockfile_version": "1",
+  "packages": {
+    "nodejs@16": {
+      "allow_insecure": true,
+      "last_modified": "2023-06-29T16:20:38Z",
+      "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#nodejs_16",
+      "source": "devbox-search",
+      "version": "16.20.1"
+    }
+  }
+}

internal/boxcli/add.go

@@ -18,7 +18,8 @@ import (
 const toSearchForPackages = "To search for packages, use the `devbox search` command"
 
 type addCmdFlags struct {
-	config configFlags
+	config        configFlags
+	allowInsecure bool
 }
 
 func addCmd() *cobra.Command {
@@ -47,13 +48,18 @@ func addCmd() *cobra.Command {
 	}
 
 	flags.config.register(command)
+	command.Flags().BoolVar(
+		&flags.allowInsecure, "allow-insecure", false,
+		"Allow adding packages marked as insecure.")
+
 	return command
 }
 
 func addCmdFunc(cmd *cobra.Command, args []string, flags addCmdFlags) error {
 	box, err := devbox.Open(&devopt.Opts{
-		Dir:    flags.config.path,
-		Writer: cmd.ErrOrStderr(),
+		Dir:               flags.config.path,
+		Writer:            cmd.ErrOrStderr(),
+		AllowInsecureAdds: flags.allowInsecure,
 	})
 	if err != nil {
 		return errors.WithStack(err)

internal/devpkg/package.go

@@ -542,3 +542,23 @@ func (p *Package) ContentAddressedPath() (string, error) {
 	}
 	return localPath, err
 }
+
+func (p *Package) AllowInsecure() bool {
+	return p.lockfile.Get(p.Raw).IsAllowInsecure()
+}
+
+// StoreName returns the last section of the store path. Example:
+// /nix/store/abc123-foo-1.0.0 -> foo-1.0.0
+// Warning, this is probably slowish. If you need to call this multiple times,
+// consider caching the result.
+func (p *Package) StoreName() (string, error) {
+	u, err := p.urlForInstall()
+	if err != nil {
+		return "", err
+	}
+	name, err := nix.EvalPackageName(u)
+	if err != nil {
+		return "", err
+	}
+	return name, nil
+}

internal/impl/devbox.go

@@ -52,12 +52,13 @@ const (
 )
 
 type Devbox struct {
-	cfg           *devconfig.Config
-	lockfile      *lock.File
-	nix           nix.Nixer
-	projectDir    string
-	pluginManager *plugin.Manager
-	pure          bool
+	cfg               *devconfig.Config
+	lockfile          *lock.File
+	nix               nix.Nixer
+	projectDir        string
+	pluginManager     *plugin.Manager
+	pure              bool
+	allowInsecureAdds bool
 
 	// Possible TODO: hardcode this to stderr. Allowing the caller to specify the
 	// writer is error prone. Since it is almost always stderr, we should default
@@ -81,18 +82,24 @@ func Open(opts *devopt.Opts) (*Devbox, error) {
 	}
 
 	box := &Devbox{
-		cfg:           cfg,
-		nix:           &nix.Nix{},
-		projectDir:    projectDir,
-		pluginManager: plugin.NewManager(),
-		writer:        opts.Writer,
-		pure:          opts.Pure,
+		cfg:               cfg,
+		nix:               &nix.Nix{},
+		projectDir:        projectDir,
+		pluginManager:     plugin.NewManager(),
+		writer:            opts.Writer,
+		pure:              opts.Pure,
+		allowInsecureAdds: opts.AllowInsecureAdds,
 	}
 
 	lock, err := lock.GetFile(box)
 	if err != nil {
 		return nil, err
 	}
+	// if lockfile has any allow insecure, we need to set the env var to ensure
+	// all nix commands work.
+	if opts.AllowInsecureAdds || lock.HasAllowInsecurePackages() {
+		nix.AllowInsecurePackages()
+	}
 	box.pluginManager.ApplyOptions(
 		plugin.WithDevbox(box),
 		plugin.WithLockfile(lock),

internal/impl/devopt/devboxopts.go

@@ -5,8 +5,9 @@ import (
 )
 
 type Opts struct {
-	Dir            string
-	Pure           bool
-	IgnoreWarnings bool
-	Writer         io.Writer
+	AllowInsecureAdds bool
+	Dir               string
+	Pure              bool
+	IgnoreWarnings    bool
+	Writer            io.Writer
 }

internal/impl/packages.go

@@ -40,10 +40,16 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
 
 	// Only add packages that are not already in config. If same canonical exists,
 	// replace it.
-	pkgs := []*devpkg.Package{}
-	for _, pkg := range devpkg.PackageFromStrings(lo.Uniq(pkgsNames), d.lockfile) {
+	pkgs := devpkg.PackageFromStrings(lo.Uniq(pkgsNames), d.lockfile)
+
+	// addedPackageNames keeps track of the possibly transformed (versioned)
+	// names of added packages (even if they are already in config). We use this
+	// to know the exact name to mark as allowed insecure later on.
+	addedPackageNames := []string{}
+	for _, pkg := range pkgs {
 		// If exact versioned package is already in the config, skip.
 		if slices.Contains(d.cfg.Packages, pkg.Versioned()) {
+			addedPackageNames = append(addedPackageNames, pkg.Versioned())
 			continue
 		}
 
@@ -61,11 +67,30 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
 		// if not, fallback to legacy vanilla nix.
 		versionedPkg := devpkg.PackageFromString(pkg.Versioned(), d.lockfile)
 		ok, err := versionedPkg.ValidateExists()
+		packageNameForConfig := pkg.Raw
 		if err == nil && ok {
-			d.cfg.Packages = append(d.cfg.Packages, pkg.Versioned())
-		} else {
-			// fallthrough and treat package as a legacy package.
-			d.cfg.Packages = append(d.cfg.Packages, pkg.Raw)
+			// Only use versioned if it exists in search.
+			packageNameForConfig = pkg.Versioned()
+		}
+		// else {
+		// 	// TODO (landau): use nix.Search to check if this package exists
+		// 	// fallthrough and treat package as a legacy package.
+		// }
+
+		d.cfg.Packages = append(d.cfg.Packages, packageNameForConfig)
+		addedPackageNames = append(addedPackageNames, packageNameForConfig)
+	}
+
+	// Resolving here ensures we allow insecure before running ensurePackagesAreInstalled
+	// which will call print-dev-env. Resolving does not save the lockfile, we
+	// save at the end when everything has succeeded.
+	if d.allowInsecureAdds {
+		for _, name := range addedPackageNames {
+			p, err := d.lockfile.Resolve(name)
+			if err != nil {
+				return err
+			}
+			p.AllowInsecure = true
 		}
 	}
 
@@ -88,9 +113,7 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
 		}
 	}
 
-	if err := d.lockfile.Add(
-		lo.Map(pkgs, func(pkg *devpkg.Package, _ int) string { return pkg.Raw })...,
-	); err != nil {
+	if err := d.lockfile.Save(); err != nil {
 		return err
 	}
 

internal/lock/lockfile.go

@@ -113,6 +113,15 @@ func (l *File) Get(pkg string) *Package {
 	return entry
 }
 
+func (l *File) HasAllowInsecurePackages() bool {
+	for _, pkg := range l.Packages {
+		if pkg.AllowInsecure {
+			return true
+		}
+	}
+	return false
+}
+
 // This probably belongs in input.go but can't add it there because it will
 // create a circular dependency. We could move Input into own package.
 func IsLegacyPackage(pkg string) bool {

internal/lock/package.go

@@ -9,6 +9,7 @@ const (
 )
 
 type Package struct {
+	AllowInsecure bool   `json:"allow_insecure,omitempty"`
 	LastModified  string `json:"last_modified,omitempty"`
 	PluginVersion string `json:"plugin_version,omitempty"`
 	Resolved      string `json:"resolved,omitempty"`
@@ -35,3 +36,10 @@ func (p *Package) GetSource() string {
 	}
 	return p.Source
 }
+
+func (p *Package) IsAllowInsecure() bool {
+	if p == nil {
+		return false
+	}
+	return p.AllowInsecure
+}

internal/nix/command.go

@@ -1,7 +1,6 @@
 package nix
 
 import (
-	"os"
 	"os/exec"
 )
 
@@ -12,6 +11,10 @@ func command(args ...string) *exec.Cmd {
 	return cmd
 }
 
-func allowUnfreeEnv() []string {
-	return append(os.Environ(), "NIXPKGS_ALLOW_UNFREE=1")
+func allowUnfreeEnv(curEnv []string) []string {
+	return append(curEnv, "NIXPKGS_ALLOW_UNFREE=1")
+}
+
+func allowInsecureEnv(curEnv []string) []string {
+	return append(curEnv, "NIXPKGS_ALLOW_INSECURE=1")
 }

internal/nix/eval.go

@@ -0,0 +1,56 @@
+package nix
+
+import (
+	"encoding/json"
+	"os"
+	"strconv"
+)
+
+func EvalPackageName(path string) (string, error) {
+	cmd := command("eval", "--raw", path+".name")
+	out, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return string(out), nil
+}
+
+// PackageIsInsecure is a fun little nix eval that maybe works.
+func PackageIsInsecure(path string) bool {
+	cmd := command("eval", path+".meta.insecure")
+	out, err := cmd.Output()
+	if err != nil {
+		// We can't know for sure, but probably not.
+		return false
+	}
+	var insecure bool
+	if err := json.Unmarshal(out, &insecure); err != nil {
+		// We can't know for sure, but probably not.
+		return false
+	}
+	return insecure
+}
+
+func PackageKnownVulnerabilities(path string) []string {
+	cmd := command("eval", path+".meta.knownVulnerabilities")
+	out, err := cmd.Output()
+	if err != nil {
+		// We can't know for sure, but probably not.
+		return nil
+	}
+	var vulnerabilities []string
+	if err := json.Unmarshal(out, &vulnerabilities); err != nil {
+		// We can't know for sure, but probably not.
+		return nil
+	}
+	return vulnerabilities
+}
+
+func AllowInsecurePackages() {
+	os.Setenv("NIXPKGS_ALLOW_INSECURE", "1")
+}
+
+func IsInsecureAllowed() bool {
+	allowed, _ := strconv.ParseBool(os.Getenv("NIXPKGS_ALLOW_INSECURE"))
+	return allowed
+}

internal/nix/nix.go

@@ -10,11 +10,13 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime/trace"
 	"strings"
 
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
+	"go.jetpack.io/devbox/internal/boxcli/usererr"
 
 	"go.jetpack.io/devbox/internal/debug"
 )
@@ -69,7 +71,9 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
 		data, err = cmd.Output()
-		if err != nil {
+		if insecure, insecureErr := isExitErrorInsecurePackage(err); insecure {
+			return nil, insecureErr
+		} else if err != nil {
 			return nil, errors.Wrapf(err, "Command: %s", cmd)
 		}
 
@@ -120,6 +124,7 @@ func System() (string, error) {
 	// For Savil to debug "remove nixpkgs" feature. The Search api lacks x86-darwin info.
 	// So, I need to fake that I am x86-linux and inspect the output in generated devbox.lock
 	// and flake.nix files.
+	// This is also used by unit tests.
 	override := os.Getenv("__DEVBOX_NIX_SYSTEM")
 	if override != "" {
 		return override, nil
@@ -144,3 +149,19 @@ func System() (string, error) {
 func ProfileBinPath(projectDir string) string {
 	return filepath.Join(projectDir, ProfilePath, "bin")
 }
+
+func isExitErrorInsecurePackage(err error) (bool, error) {
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
+		if strings.Contains(string(exitErr.Stderr), "is marked as insecure") {
+			re := regexp.MustCompile(`Package ([^ ]+)`)
+			match := re.FindStringSubmatch(string(exitErr.Stderr))
+			return true, usererr.New(
+				"Package %s is insecure. \n\n"+
+					"To override use `devbox add <pkg> --allow-insecure`",
+				match[0],
+			)
+		}
+	}
+	return false, nil
+}