[plugins] Allow local non-compiled plugins

examples/plugins/local/README.md

@@ -0,0 +1,7 @@
+# Custom plugin example
+
+Shows how to write custom local plugin. Plugins can:
+
+* Install packages
+* Create templatized files (including flakes)
+* Declare services (using process-compose)

examples/plugins/local/devbox.json

@@ -0,0 +1,16 @@
+{
+  "packages": [],
+  "shell": {
+    "init_hook": [
+      "echo 'Welcome to devbox!' > /dev/null"
+    ],
+    "scripts": {
+      "run_test": [
+        "./test.sh"
+      ]
+    }
+  },
+  "include": [
+    "path:my-plugin/my-plugin.json"
+  ]
+}

examples/plugins/local/devbox.lock

@@ -0,0 +1,6 @@
+{
+  "lockfile_version": "1",
+  "packages": {
+    "path:my-plugin/my-plugin.json": {}
+  }
+}

examples/plugins/local/my-plugin/my-plugin.json

@@ -0,0 +1,13 @@
+{
+  "name": "my-plugin",
+  "version": "0.0.1",
+  "readme": "Example custom plugin",
+  "env": {
+    "MY_FOO_VAR": "BAR"
+  },
+  "create_files": {
+    "{{ .Virtenv }}/empty-dir": "",
+    "{{ .Virtenv }}/some-file": "some-file.txt",
+    "{{ .Virtenv }}/process-compose.yaml": "process-compose.yaml"
+  }
+}

examples/plugins/local/my-plugin/process-compose.yaml

@@ -0,0 +1,7 @@
+version: "0.5"
+
+processes:
+  my-plugin-service:
+    command: echo "success" && tail -f /dev/null
+    availability:
+      restart: "always"

examples/plugins/local/my-plugin/some-file.txt

@@ -0,0 +1 @@
+some data

examples/plugins/local/test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "$MY_FOO_VAR" ]; then
+  echo "MY_FOO_VAR environment variable is not set."
+  exit 1
+else
+ echo "MY_FOO_VAR is set to '$MY_FOO_VAR'"
+fi

internal/impl/devbox.go

@@ -127,12 +127,15 @@ func (d *Devbox) Config() *devconfig.Config {
 }
 
 func (d *Devbox) ConfigHash() (string, error) {
-	hashes := lo.Map(d.PackagesAsInputs(), func(i *devpkg.Package, _ int) string { return i.Hash() })
+	pkgHashes := lo.Map(d.PackagesAsInputs(), func(i *devpkg.Package, _ int) string { return i.Hash() })
+	includeHashes := lo.Map(d.Includes(), func(i plugin.Includable, _ int) string { return i.Hash() })
 	h, err := d.cfg.Hash()
 	if err != nil {
 		return "", err
 	}
-	return cuecfg.Hash(h + strings.Join(hashes, ""))
+	return cuecfg.Hash(
+		h + strings.Join(pkgHashes, "") + strings.Join(includeHashes, ""),
+	)
 }
 
 func (d *Devbox) NixPkgsCommitHash() string {
@@ -919,6 +922,16 @@ func (d *Devbox) PackagesAsInputs() []*devpkg.Package {
 	return devpkg.PackageFromStrings(d.Packages(), d.lockfile)
 }
 
+func (d *Devbox) Includes() []plugin.Includable {
+	includes := []plugin.Includable{}
+	for _, includePath := range d.cfg.Include {
+		if include, err := d.pluginManager.ParseInclude(includePath); err == nil {
+			includes = append(includes, include)
+		}
+	}
+	return includes
+}
+
 func (d *Devbox) HasDeprecatedPackages() bool {
 	for _, pkg := range d.PackagesAsInputs() {
 		if pkg.IsLegacy() {

internal/plugin/files.go

@@ -4,20 +4,28 @@
 package plugin
 
 import (
+	"os"
 	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
-	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/plugins"
 )
 
-func getConfigIfAny(pkg *devpkg.Package, projectDir string) (*config, error) {
+func getConfigIfAny(pkg Includable, projectDir string) (*config, error) {
 	configFiles, err := plugins.BuiltIn.ReadDir(".")
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
+	if local, ok := pkg.(*localPlugin); ok {
+		content, err := os.ReadFile(local.path)
+		if err != nil && !os.IsNotExist(err) {
+			return nil, errors.WithStack(err)
+		}
+		return buildConfig(pkg, projectDir, string(content))
+	}
+
 	for _, file := range configFiles {
 		if file.IsDir() || strings.HasSuffix(file.Name(), ".go") {
 			continue
@@ -44,6 +52,9 @@ func getConfigIfAny(pkg *devpkg.Package, projectDir string) (*config, error) {
 	return nil, nil
 }
 
-func getFileContent(contentPath string) ([]byte, error) {
+func getFileContent(pkg Includable, contentPath string) ([]byte, error) {
+	if local, ok := pkg.(*localPlugin); ok {
+		return os.ReadFile(local.contentPath(contentPath))
+	}
 	return plugins.BuiltIn.ReadFile(contentPath)
 }

internal/plugin/includes.go

@@ -1,18 +1,81 @@
 package plugin
 
 import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"regexp"
 	"strings"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
+	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/devpkg"
 )
 
-func (m *Manager) parseInclude(include string) (*devpkg.Package, error) {
+type Includable interface {
+	CanonicalName() string
+	Hash() string
+}
+
+func (m *Manager) ParseInclude(include string) (Includable, error) {
 	includeType, name, _ := strings.Cut(include, ":")
-	if includeType != "plugin" {
-		return nil, usererr.New("unknown include type %q", includeType)
-	} else if name == "" {
+	if name == "" {
 		return nil, usererr.New("include name is required")
+	} else if includeType == "plugin" {
+		return devpkg.PackageFromString(name, m.lockfile), nil
+	} else if includeType == "path" {
+		absPath := filepath.Join(m.ProjectDir(), name)
+		return newLocalPlugin(absPath)
+	}
+	return nil, usererr.New("unknown include type %q", includeType)
+}
+
+type localPlugin struct {
+	name string
+	path string
+}
+
+var nameRegex = regexp.MustCompile(`^[a-zA-Z0-9_\- ]+$`)
+
+func newLocalPlugin(path string) (*localPlugin, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	m := map[string]any{}
+	if err := json.Unmarshal(content, &m); err != nil {
+		return nil, err
+	}
+	name, ok := m["name"].(string)
+	if !ok || name == "" {
+		return nil,
+			usererr.New("plugin %s is missing a required field 'name'", path)
 	}
-	return devpkg.PackageFromString(name, m.lockfile), nil
+	if !nameRegex.MatchString(name) {
+		return nil, usererr.New(
+			"plugin %s has an invalid name %q. Name must match %s",
+			path, name, nameRegex,
+		)
+	}
+	return &localPlugin{
+		name: name,
+		path: path,
+	}, nil
+}
+
+func (l *localPlugin) CanonicalName() string {
+	return l.name
+}
+
+func (l *localPlugin) IsLocal() bool {
+	return true
+}
+
+func (l *localPlugin) contentPath(subpath string) string {
+	return filepath.Join(filepath.Dir(l.path), subpath)
+}
+
+func (l *localPlugin) Hash() string {
+	h, _ := cuecfg.FileHash(l.path)
+	return h
 }

internal/plugin/plugin.go

@@ -68,7 +68,7 @@ func (c *config) Services() (services.Services, error) {
 }
 
 func (m *Manager) Include(included string) error {
-	name, err := m.parseInclude(included)
+	name, err := m.ParseInclude(included)
 	if err != nil {
 		return err
 	}
@@ -80,7 +80,7 @@ func (m *Manager) Create(pkg *devpkg.Package) error {
 	return m.create(pkg, m.lockfile.Packages[pkg.Raw])
 }
 
-func (m *Manager) create(pkg *devpkg.Package, locked *lock.Package) error {
+func (m *Manager) create(pkg Includable, locked *lock.Package) error {
 	virtenvPath := filepath.Join(m.ProjectDir(), VirtenvPath)
 	cfg, err := getConfigIfAny(pkg, m.ProjectDir())
 	if err != nil {
@@ -129,12 +129,12 @@ func (m *Manager) create(pkg *devpkg.Package, locked *lock.Package) error {
 }
 
 func (m *Manager) createFile(
-	pkg *devpkg.Package,
+	pkg Includable,
 	filePath, contentPath, virtenvPath string,
 ) error {
 	name := pkg.CanonicalName()
 	debug.Log("Creating file %q from contentPath: %q", filePath, contentPath)
-	content, err := getFileContent(contentPath)
+	content, err := getFileContent(pkg, contentPath)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -148,9 +148,14 @@ func (m *Manager) createFile(
 		return err
 	}
 
-	attributePath, err := pkg.PackageAttributePath()
-	if err != nil {
-		return err
+	var urlForInput, attributePath string
+
+	if pkg, ok := pkg.(*devpkg.Package); ok {
+		attributePath, err = pkg.PackageAttributePath()
+		if err != nil {
+			return err
+		}
+		urlForInput = pkg.URLForFlakeInput()
 	}
 
 	var buf bytes.Buffer
@@ -162,7 +167,7 @@ func (m *Manager) createFile(
 		"PackageAttributePath": attributePath,
 		"Packages":             m.Packages(),
 		"System":               system,
-		"URLForInput":          pkg.URLForFlakeInput(),
+		"URLForInput":          urlForInput,
 		"Virtenv":              filepath.Join(virtenvPath, name),
 	}); err != nil {
 		return errors.WithStack(err)
@@ -193,9 +198,12 @@ func (m *Manager) Env(
 	includes []string,
 	computedEnv map[string]string,
 ) (map[string]string, error) {
-	allPkgs := append([]*devpkg.Package(nil), pkgs...)
+	allPkgs := []Includable{}
+	for _, pkg := range pkgs {
+		allPkgs = append(allPkgs, pkg)
+	}
 	for _, included := range includes {
-		input, err := m.parseInclude(included)
+		input, err := m.ParseInclude(included)
 		if err != nil {
 			return nil, err
 		}
@@ -218,7 +226,7 @@ func (m *Manager) Env(
 	return conf.OSExpandEnvMap(env, computedEnv, m.ProjectDir()), nil
 }
 
-func buildConfig(pkg *devpkg.Package, projectDir, content string) (*config, error) {
+func buildConfig(pkg Includable, projectDir, content string) (*config, error) {
 	cfg := &config{}
 	name := pkg.CanonicalName()
 	t, err := template.New(name + "-template").Parse(content)

internal/plugin/services.go

@@ -17,9 +17,12 @@ func (m *Manager) GetServices(
 ) (services.Services, error) {
 	allSvcs := services.Services{}
 
-	allPkgs := append([]*devpkg.Package(nil), pkgs...)
+	allPkgs := []Includable{}
+	for _, pkg := range pkgs {
+		allPkgs = append(allPkgs, pkg)
+	}
 	for _, include := range includes {
-		name, err := m.parseInclude(include)
+		name, err := m.ParseInclude(include)
 		if err != nil {
 			return nil, err
 		}