[rm nixpkgs] use path-from-hash-part command when updating lock file (#1442)

## Summary

**Motivation**
With Remove Nixpkgs feature on, installing `[email protected]` was failing with:
```
[3/7] [email protected]
        don't know how to build these paths:
          /nix/store/0xyi7w3cm3g7gzwfpwqzx0n7xks2sdc6-curl-8.0.1
        asked 'https://cache.nixos.org/' for '/nix/store/0xyi7w3cm3g7gzwfpwqzx0n7xks2sdc6-curl-8.0.1' but got '/nix/store/0xyi7w3cm3g7gzwfpwqzx0n7xks2sdc6-curl-8.0.1-bin'
        error: path '/nix/store/0xyi7w3cm3g7gzwfpwqzx0n7xks2sdc6-curl-8.0.1' is required, but there is no substituter that can build it
```

**Explanation**
Thanks to @gcurtis.
curl’s first (and therefore default) output is `bin`, not `out`, so Nix
is appending `-bin` to the path.

From `man nix-build`:
> If a derivation has multiple outputs, nix-build will build the default
(first) output. You can also build all outputs:
$ nix-build '<nixpkgs>' --attr openssl.all
This will create a symlink for each output named result-outputname. The
suffix is omitted if the output name is out. So if openssl has outputs
out, bin and man, nix-build will create symlinks result, result-bin and
result-man.

**Fix**
Observe:
```
> nix store path-from-hash-part --store https://cache.nixos.org/ bbkz21vmx16wl0h3xk36g7gxpbymnm6d --extra-experimental-features "nix-command flakes"
/nix/store/bbkz21vmx16wl0h3xk36g7gxpbymnm6d-curl-8.0.1-bin
```

So:
- During `devbox update`, we derive the store-path from the search API
and write to `devbox.lock`
- Previously, we would manually construct store-path from
`hash-name-version` from search API response
- Now, we invoke `nix store path-from-hash-part <hash>` and write the
response store-path in the `devbox.lock`.
- if this fails (it shouldn't!), we skip writing the store-path to the
`devbox.lock`, and print a warning. Maybe we should track it in sentry?
  

## How was it tested?

```
> cat devbox.json
{
  "packages": [
    "[email protected]"
  ]
}

> devbox update
Info: Updated system information for [email protected]
Ensuring packages are installed.

Installing package: [email protected].

[1/1] [email protected]
[1/1] [email protected]: Success
Info: Running "nix flake update"

> cat devbox.lock
{
  "lockfile_version": "1",
  "packages": {
    "[email protected]": {
      "last_modified": "2023-05-31T02:09:55Z",
      "resolved": "github:NixOS/nixpkgs/9cfaa8a1a00830d17487cb60a19bb86f96f09b27#curl",
      "source": "devbox-search",
      "version": "8.0.1",
      "systems": {
        "aarch64-darwin": {
          "store_path": "/nix/store/7qwsslwdrjyg8hs654nsqpq7lly6dhq5-curl-8.0.1-bin"
        },
        "aarch64-linux": {
          "store_path": "/nix/store/g9g1p4l9146kclnx4lv0hfbzhr9hk7n7-curl-8.0.1-bin"
        }
      }
    }
  }
}
```

Author: savil

internal/lock/resolve.go

@@ -4,16 +4,22 @@
 package lock
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/searcher"
+	"go.jetpack.io/devbox/internal/ux"
 	"golang.org/x/exp/maps"
+	"golang.org/x/sync/errgroup"
 )
 
 // FetchResolvedPackage fetches a resolution but does not write it to the lock
@@ -35,7 +41,10 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
 
 	sysInfos := map[string]*SystemInfo{}
 	if featureflag.RemoveNixpkgs.Enabled() {
-		sysInfos = buildLockSystemInfos(packageVersion)
+		sysInfos, err = buildLockSystemInfos(packageVersion)
+		if err != nil {
+			return nil, err
+		}
 	}
 	packageInfo, err := selectForSystem(packageVersion)
 	if err != nil {
@@ -74,20 +83,57 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error)
 	return maps.Values(pkg.Systems)[0], nil
 }
 
-func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo {
-	sysInfos := map[string]*SystemInfo{}
-	for sysName, sysInfo := range pkg.Systems {
+func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) {
+	// guard against missing search data
+	systems := lo.PickBy(pkg.Systems, func(sysName string, sysInfo searcher.PackageInfo) bool {
+		return sysInfo.StoreHash != "" && sysInfo.StoreName != ""
+	})
 
-		// guard against missing search data
-		if sysInfo.StoreHash == "" || sysInfo.StoreName == "" {
-			debug.Log("WARN: skipping %s in %s due to missing store name or hash", pkg.Name, sysName)
-			continue
-		}
+	group, ctx := errgroup.WithContext(context.Background())
+
+	var storePathLock sync.RWMutex
+	sysStorePaths := map[string]string{}
+	for _sysName, _sysInfo := range systems {
+		sysName := _sysName // capture range variable
+		sysInfo := _sysInfo // capture range variable
+
+		group.Go(func() error {
+			// We should use devpkg.BinaryCache here, but it'll cause a circular reference
+			// Just hardcoding for now. Maybe we should move that to nix.DefaultBinaryCache?
+			path, err := nix.StorePathFromHashPart(ctx, sysInfo.StoreHash, "https://cache.nixos.org")
+			if err != nil {
+				// Should we report this to sentry to collect data?
+				ux.Fwarning(
+					os.Stderr,
+					"Failed to resolve store path for %s with storeHash %s. Installing will be a bit slower.\n",
+					sysName,
+					sysInfo.StoreHash,
+				)
+				debug.Log(
+					"Failed to resolve store path for %s with storeHash %s. Error is %s.\n",
+					sysName,
+					sysInfo.StoreHash,
+					err,
+				)
+				// Instead of erroring, we can just skip this package. It can install via the slow path.
+				return nil
+			}
+			storePathLock.Lock()
+			sysStorePaths[sysName] = path
+			storePathLock.Unlock()
+			return nil
+		})
+	}
+	err := group.Wait()
+	if err != nil {
+		return nil, err
+	}
 
-		storePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion)
+	sysInfos := map[string]*SystemInfo{}
+	for sysName, storePath := range sysStorePaths {
 		sysInfos[sysName] = &SystemInfo{
 			StorePath: storePath,
 		}
 	}
-	return sysInfos
+	return sysInfos, nil
 }

internal/nix/command.go

@@ -1,12 +1,16 @@
 package nix
 
 import (
+	"context"
 	"os/exec"
 )
 
 func command(args ...string) *exec.Cmd {
+	return commandContext(context.Background(), args...)
+}
 
-	cmd := exec.Command("nix", args...)
+func commandContext(ctx context.Context, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, "nix", args...)
 	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
 	return cmd
 }

internal/nix/store.go

@@ -1,15 +1,15 @@
 package nix
 
 import (
-	"path/filepath"
+	"context"
 	"strings"
 )
 
-func StorePath(hash, name, version string) string {
-	storeDirParts := []string{hash, name}
-	if version != "" {
-		storeDirParts = append(storeDirParts, version)
+func StorePathFromHashPart(ctx context.Context, hash, storeAddr string) (string, error) {
+	cmd := commandContext(ctx, "store", "path-from-hash-part", "--store", storeAddr, hash)
+	resultBytes, err := cmd.Output()
+	if err != nil {
+		return "", err
 	}
-	storeDir := strings.Join(storeDirParts, "-")
-	return filepath.Join("/nix/store", storeDir)
+	return strings.TrimSpace(string(resultBytes)), nil
 }