devpkg: better flake references and installable parsing (#1581)

In anticipation of adding support for non-default package outputs, I
wanted to clean up some of the flakeref parsing in `devpkg`. This will
make it easier to handle Nix's `flakeref#attrpath^outputs` installable
syntax. This PR is broken up into two commits: one that handles flakeref
parsing and one that handles installable parsing.

The `ParseFlakeRef` function is able to parse the more common flakerefs
that devbox currently supports ("path", "flake", "indirect", etc.). It
doesn't support mercurial, gitlab, and sourcehuthut for now. The
`ParseFlakeInstallable` function parses Nix flake installables, which
are a superset of the flake reference syntax. It allows an attribute
path and/or outputs to be specified using the `flake#attrpath^outputs`
syntax. The tests were validated using the nix expression
`builtins.parseFlakeRef "ref"` and the `nix flake metadata <ref>` CLI.

The goal isn't to perfectly replicate Nix's parsing of flake references
(which itself is sometimes inconsistent), but to make it good enough to
handle flakerefs in devbox.json.

Author: gcurtis

internal/devpkg/flakeref.go

@@ -0,0 +1,356 @@
+package devpkg
+
+import (
+	"net/url"
+	"strings"
+
+	"go.jetpack.io/devbox/internal/redact"
+)
+
+// FlakeRef is a parsed Nix flake reference. A flake reference is a subset of
+// the Nix CLI "installable" syntax. Installables may specify an attribute path
+// and derivation outputs with a flake reference using the '#' and '^' characters.
+// For example, the string "nixpkgs" and "./flake" are valid flake references,
+// but "nixpkgs#hello" and "./flake#app^bin,dev" are not.
+//
+// See the [Nix manual] for details on flake references.
+//
+// [Nix manual]: https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix3-flake
+type FlakeRef struct {
+	// Type is the type of flake reference. Some valid types are "indirect",
+	// "path", "file", "git", "tarball", and "github".
+	Type string `json:"type,omitempty"`
+
+	// ID is the flake's identifier when Type is "indirect". A common
+	// example is nixpkgs.
+	ID string `json:"id,omitempty"`
+
+	// Path is the path to the flake directory when Type is "path".
+	Path string `json:"path,omitempty"`
+
+	// Owner and repo are the flake repository owner and name when Type is
+	// "github".
+	Owner string `json:"owner,omitempty"`
+	Repo  string `json:"repo,omitempty"`
+
+	// Rev and ref are the git revision (commit hash) and ref
+	// (branch or tag) when Type is "github" or "git".
+	Rev string `json:"rev,omitempty"`
+	Ref string `json:"ref,omitempty"`
+
+	// Dir is non-empty when the directory containing the flake.nix file is
+	// not at the flake root. It corresponds to the optional "dir" query
+	// parameter when Type is "github", "git", "tarball", or "file".
+	Dir string `json:"dir,omitempty"`
+
+	// Host overrides the default VCS host when Type is "github", such as
+	// when referring to a GitHub Enterprise instance. It corresponds to the
+	// optional "host" query parameter when Type is "github".
+	Host string `json:"host,omitempty"`
+
+	// URL is the URL pointing to the flake when type is "tarball", "file",
+	// or "git". Note that the URL is not the same as the raw unparsed
+	// flakeref.
+	URL string `json:"url,omitempty"`
+
+	// raw stores the original unparsed flakeref string.
+	raw string
+}
+
+// ParseFlakeRef parses a raw flake reference. Nix supports a variety of
+// flakeref formats, and isn't entirely consistent about how it parses them.
+// ParseFlakeRef attempts to mimic how Nix parses flakerefs on the command line.
+// The raw ref can be one of the following:
+//
+//   - Indirect reference such as "nixpkgs" or "nixpkgs/unstable".
+//   - Path-like reference such as "./flake" or "/path/to/flake". They must
+//     start with a '.' or '/' and not contain a '#' or '?'.
+//   - URL-like reference which must be a valid URL with any special characters
+//     encoded. The scheme can be any valid flakeref type except for mercurial,
+//     gitlab, and sourcehut.
+//
+// ParseFlakeRef does not guarantee that a parsed flakeref is valid or that an
+// error indicates an invalid flakeref. Use the "nix flake metadata" command or
+// the parseFlakeRef builtin function to validate a flakeref.
+func ParseFlakeRef(ref string) (FlakeRef, error) {
+	if ref == "" {
+		return FlakeRef{}, redact.Errorf("empty flake reference")
+	}
+
+	// Handle path-style references first.
+	parsed := FlakeRef{raw: ref}
+	if ref[0] == '.' || ref[0] == '/' {
+		if strings.ContainsAny(ref, "?#") {
+			// The Nix CLI does seem to allow paths with a '?'
+			// (contrary to the manual) but ignores everything that
+			// comes after it. This is a bit surprising, so we just
+			// don't allow it at all.
+			return FlakeRef{}, redact.Errorf("path-style flake reference %q contains a '?' or '#'", ref)
+		}
+		parsed.Type = "path"
+		parsed.Path = ref
+		return parsed, nil
+	}
+	parsed, _, err := parseFlakeURLRef(ref)
+	return parsed, err
+}
+
+func parseFlakeURLRef(ref string) (parsed FlakeRef, fragment string, err error) {
+	// A good way to test how Nix parses a flake reference is to run:
+	//
+	// 	nix eval --json --expr 'builtins.parseFlakeRef "ref"' | jq
+	parsed.raw = ref
+	refURL, err := url.Parse(ref)
+	if err != nil {
+		return FlakeRef{}, "", redact.Errorf("parse flake reference as URL: %v", err)
+	}
+
+	switch refURL.Scheme {
+	case "", "flake":
+		// [flake:]<flake-id>(/<rev-or-ref>(/rev)?)?
+
+		parsed.Type = "indirect"
+
+		// "indirect" is parsed as a path, "flake:indirect" is parsed as
+		// opaque because it has a scheme.
+		path := refURL.Path
+		if path == "" {
+			path, err = url.PathUnescape(refURL.Opaque)
+			if err != nil {
+				path = refURL.Opaque
+			}
+		}
+		split := strings.SplitN(path, "/", 3)
+		parsed.ID = split[0]
+		if len(split) > 1 {
+			if isGitHash(split[1]) {
+				parsed.Rev = split[1]
+			} else {
+				parsed.Ref = split[1]
+			}
+		}
+		if len(split) > 2 && parsed.Rev == "" {
+			parsed.Rev = split[2]
+		}
+	case "path":
+		// [path:]<path>(\?<params)?
+
+		parsed.Type = "path"
+		if refURL.Path == "" {
+			parsed.Path, err = url.PathUnescape(refURL.Opaque)
+			if err != nil {
+				parsed.Path = refURL.Opaque
+			}
+		} else {
+			parsed.Path = refURL.Path
+		}
+	case "http", "https", "file":
+		if isArchive(refURL.Path) {
+			parsed.Type = "tarball"
+		} else {
+			parsed.Type = "file"
+		}
+		parsed.URL = ref
+		parsed.Dir = refURL.Query().Get("dir")
+	case "tarball+http", "tarball+https", "tarball+file":
+		parsed.Type = "tarball"
+		parsed.Dir = refURL.Query().Get("dir")
+		parsed.URL = ref[8:] // remove tarball+
+	case "file+http", "file+https", "file+file":
+		parsed.Type = "file"
+		parsed.Dir = refURL.Query().Get("dir")
+		parsed.URL = ref[5:] // remove file+
+	case "git", "git+http", "git+https", "git+ssh", "git+git", "git+file":
+		parsed.Type = "git"
+		q := refURL.Query()
+		parsed.Dir = q.Get("dir")
+		parsed.Ref = q.Get("ref")
+		parsed.Rev = q.Get("rev")
+
+		// ref and rev get stripped from the query parameters, but dir
+		// stays.
+		q.Del("ref")
+		q.Del("rev")
+		refURL.RawQuery = q.Encode()
+		if len(refURL.Scheme) > 3 {
+			refURL.Scheme = refURL.Scheme[4:] // remove git+
+		}
+		parsed.URL = refURL.String()
+	case "github":
+		if err := parseGitHubFlakeRef(refURL, &parsed); err != nil {
+			return FlakeRef{}, "", err
+		}
+	}
+	return parsed, refURL.Fragment, nil
+}
+
+func parseGitHubFlakeRef(refURL *url.URL, parsed *FlakeRef) error {
+	// github:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?
+
+	parsed.Type = "github"
+	path := refURL.Path
+	if path == "" {
+		var err error
+		path, err = url.PathUnescape(refURL.Opaque)
+		if err != nil {
+			path = refURL.Opaque
+		}
+	}
+	path = strings.TrimPrefix(path, "/")
+
+	split := strings.SplitN(path, "/", 3)
+	parsed.Owner = split[0]
+	parsed.Repo = split[1]
+	if len(split) > 2 {
+		if revOrRef := split[2]; isGitHash(revOrRef) {
+			parsed.Rev = revOrRef
+		} else {
+			parsed.Ref = revOrRef
+		}
+	}
+
+	parsed.Host = refURL.Query().Get("host")
+	if qRef := refURL.Query().Get("ref"); qRef != "" {
+		if parsed.Rev != "" {
+			return redact.Errorf("github flake reference has a ref and a rev")
+		}
+		if parsed.Ref != "" && qRef != parsed.Ref {
+			return redact.Errorf("github flake reference has a ref in the path (%q) and a ref query parameter (%q)", parsed.Ref, qRef)
+		}
+		parsed.Ref = qRef
+	}
+	if qRev := refURL.Query().Get("rev"); qRev != "" {
+		if parsed.Ref != "" {
+			return redact.Errorf("github flake reference has a ref and a rev")
+		}
+		if parsed.Rev != "" && qRev != parsed.Rev {
+			return redact.Errorf("github flake reference has a rev in the path (%q) and a rev query parameter (%q)", parsed.Rev, qRev)
+		}
+		parsed.Rev = qRev
+	}
+	return nil
+}
+
+// String returns the raw flakeref string as given to ParseFlakeRef.
+func (f FlakeRef) String() string {
+	return f.raw
+}
+
+func isGitHash(s string) bool {
+	if len(s) != 40 {
+		return false
+	}
+	for i := range s {
+		isDigit := s[i] >= '0' && s[i] <= '9'
+		isHexLetter := s[i] >= 'a' && s[i] <= 'f'
+		if !isDigit && !isHexLetter {
+			return false
+		}
+	}
+	return true
+}
+
+func isArchive(path string) bool {
+	return strings.HasSuffix(path, ".tar") ||
+		strings.HasSuffix(path, ".tar.gz") ||
+		strings.HasSuffix(path, ".tgz") ||
+		strings.HasSuffix(path, ".tar.xz") ||
+		strings.HasSuffix(path, ".tar.zst") ||
+		strings.HasSuffix(path, ".tar.bz2") ||
+		strings.HasSuffix(path, ".zip")
+}
+
+// FlakeInstallable is a Nix command line argument that specifies how to install
+// a flake. It can be a plain flake reference, or a flake reference with an
+// attribute path and/or output specification.
+//
+// Some examples are:
+//
+//   - "." installs the default attribute from the flake in the current
+//     directory.
+//   - ".#hello" installs the hello attribute from the flake in the current
+//     directory.
+//   - "nixpkgs#hello" installs the hello attribute from the nixpkgs flake.
+//   - "github:NixOS/nixpkgs/unstable#curl^lib" installs the the lib output of
+//     curl attribute from the flake on the nixpkgs unstable branch.
+//
+// The flake installable syntax is only valid in Nix command line arguments, not
+// in Nix expressions. See FlakeRef and the [Nix manual for details on the
+// differences between flake references and installables.
+//
+// [Nix manual]: https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix#installables
+type FlakeInstallable struct {
+	Ref      FlakeRef
+	AttrPath string
+	Outputs  []string
+
+	raw string
+}
+
+// ParseFlakeInstallable parses a flake installable. The string s must contain a
+// valid flake reference parsable by ParseFlakeRef, optionally followed by an
+// #attrpath and/or an ^output.
+func ParseFlakeInstallable(raw string) (FlakeInstallable, error) {
+	if raw == "" {
+		return FlakeInstallable{}, redact.Errorf("empty flake installable")
+	}
+
+	// The output spec must be parsed and removed first, otherwise it will
+	// be parsed as part of the flakeref's URL fragment.
+	install := FlakeInstallable{raw: raw}
+	before, after := splitOutputSpec(raw)
+	if after != "" {
+		install.Outputs = strings.Split(after, ",")
+	}
+	raw = before
+
+	// Interpret installables with path-style flakerefs as URLs to extract
+	// the attribute path (fragment). This means that path-style flakerefs
+	// cannot point to files with a '#' or '?' in their name, since those
+	// would be parsed as the URL fragment or query string. This mimic's
+	// Nix's CLI behavior.
+	prefix := ""
+	if raw[0] == '.' || raw[0] == '/' {
+		prefix = "path:"
+		raw = prefix + raw
+	}
+
+	var err error
+	install.Ref, install.AttrPath, err = parseFlakeURLRef(raw)
+	if err != nil {
+		return FlakeInstallable{}, err
+	}
+	// Make sure to reset the raw flakeref to the original string
+	// after parsing.
+	install.Ref.raw = raw[len(prefix):]
+	return install, nil
+}
+
+// AllOutputs returns true if the installable specifies all outputs with the
+// "^*" syntax.
+func (f FlakeInstallable) AllOutputs() bool {
+	for _, out := range f.Outputs {
+		if out == "*" {
+			return true
+		}
+	}
+	return false
+}
+
+// DefaultOutputs returns true if the installable does not specify any outputs.
+func (f FlakeInstallable) DefaultOutputs() bool {
+	return len(f.Outputs) == 0
+}
+
+// String returns the raw installable string as given to ParseFlakeInstallable.
+func (f FlakeInstallable) String() string {
+	return f.raw
+}
+
+// splitOutputSpec cuts a flake installable around the last instance of ^.
+func splitOutputSpec(s string) (before, after string) {
+	if i := strings.LastIndexByte(s, '^'); i >= 0 {
+		return s[:i], s[i+1:]
+	}
+	return s, ""
+}