Skip to content

Commit c1797d0

Browse files
committed
devconfig: allow comments and trailing commas
This allows comments and trailing comments in devbox.json as supported by hujson/JWCC. ```json5 // Top-level comment about the config. { "packages": { "go": "1.20", // Needed to decompress stuff on macOS "xz": { "version": "latest", "platforms": ["aarch64-darwin"], }, }, "shell": { "scripts": { /* A longer block comment about some build script. */ "build": "go install ./..", }, }, } ``` As a part of this change, the way the `add`/`rm` commands edit and marshal devbox.json has changed significantly. We can no longer overwrite the previous JSON, since that would lose the user's comments and formatting. There's a new `configAST` type to handle surgical edits to the JSON so that those things aren't lost. Unmarshalling the config is still done the same way. The docs in `ast.go` have more details.
1 parent 63cb177 commit c1797d0

File tree

8 files changed

+1082
-210
lines changed

8 files changed

+1082
-210
lines changed

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,14 @@ require (
3636
github.com/spf13/cobra v1.7.0
3737
github.com/spf13/pflag v1.0.5
3838
github.com/stretchr/testify v1.8.4
39+
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
3940
github.com/wk8/go-ordered-map/v2 v2.1.8
4041
github.com/zealic/go2node v0.1.0
4142
go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022
4243
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
4344
golang.org/x/mod v0.12.0
4445
golang.org/x/sync v0.3.0
46+
golang.org/x/tools v0.6.0
4547
gopkg.in/natefinch/lumberjack.v2 v2.2.1
4648
gopkg.in/yaml.v3 v3.0.1
4749
)
@@ -116,7 +118,6 @@ require (
116118
golang.org/x/sys v0.12.0 // indirect
117119
golang.org/x/term v0.12.0 // indirect
118120
golang.org/x/text v0.13.0 // indirect
119-
golang.org/x/tools v0.6.0 // indirect
120121
google.golang.org/appengine v1.6.8 // indirect
121122
google.golang.org/protobuf v1.31.0 // indirect
122123
)

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
321321
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
322322
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
323323
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
324+
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw=
325+
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8=
324326
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
325327
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
326328
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
@@ -335,8 +337,6 @@ github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBU
335337
github.com/zaffka/mongodb-boltdb-mock v0.0.0-20221014194232-b4bb03fbe3a0/go.mod h1:GsDD1qsG+86MeeCG7ndi6Ei3iGthKL3wQ7PTFigDfNY=
336338
github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA=
337339
github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg=
338-
go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3 h1:aMydtVCHn7dfotOyV41VAxX5b5OOsCc4TxOXwDt38Yw=
339-
go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA=
340340
go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022 h1:8TRpo0lYh1Y6zec8Px0yXbnVRXXs+yUPU+TnHNsREdA=
341341
go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA=
342342
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

internal/devconfig/ast.go

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
package devconfig
2+
3+
import (
4+
"bytes"
5+
"slices"
6+
7+
"github.com/tailscale/hujson"
8+
)
9+
10+
// configAST is a hujson syntax tree that represents a devbox.json
11+
// configuration. An AST allows the CLI to modify specific parts of a user's
12+
// devbox.json instead of overwriting the entire file. This is important
13+
// because a devbox.json can have user comments that must be preserved when
14+
// saving changes.
15+
//
16+
// - Unmarshalling is still done with encoding/json.
17+
// - Marshalling is done by calling configAST.root.Pack to encode the AST as
18+
// hujson/JWCC. Therefore, any changes to a Config struct will NOT
19+
// automatically be marshaled back to JSON. Support for modifying a part of
20+
// the JSON must be explicitly implemented in configAST.
21+
// - Validation with the AST is complex, so it doesn't do any. It will happily
22+
// append duplicate object keys and panic on invalid types. The higher-level
23+
// Config type is responsible for tracking state and making valid edits to
24+
// the AST.
25+
//
26+
// Be aware that there are 4 ways of representing a package in devbox.json that
27+
// the AST needs to handle:
28+
//
29+
// 1. ["name"] or ["name@version"] (versioned name array)
30+
// 2. {"name": "version"} (packages object member with version string)
31+
// 3. {"name": {"version": "1.2.3"}} (packages object member with package object)
32+
// 4. {"github:F1bonacc1/process-compose/v0.40.2": {}} (packages object member with flakeref)
33+
type configAST struct {
34+
root hujson.Value
35+
}
36+
37+
// parseConfig parses the bytes of a devbox.json and returns a syntax tree.
38+
func parseConfig(b []byte) (*configAST, error) {
39+
root, err := hujson.Parse(b)
40+
if err != nil {
41+
return nil, err
42+
}
43+
return &configAST{root: root}, nil
44+
}
45+
46+
// packagesField gets the "packages" field, initializing it if necessary. The
47+
// member value will either be an array of strings or an object. When it's an
48+
// object, the keys will always be package names and the values will be a
49+
// string or another object. Examples are:
50+
//
51+
// - {"packages": ["go", "hello"]}
52+
// - {"packages": {"go": "1.20", "hello: {"platforms": ["aarch64-darwin"]}}}
53+
//
54+
// When migrate is true, the packages value will be migrated from the legacy
55+
// array format to the object format. For example, the array:
56+
//
57+
// ["go@latest", "hello"]
58+
//
59+
// will become:
60+
//
61+
// {
62+
// "go": "latest",
63+
// "hello": ""
64+
// }
65+
func (c *configAST) packagesField(migrate bool) *hujson.ObjectMember {
66+
rootObject := c.root.Value.(*hujson.Object)
67+
i := c.memberIndex(rootObject, "packages")
68+
if i != -1 {
69+
switch rootObject.Members[i].Value.Value.Kind() {
70+
case '[':
71+
if migrate {
72+
c.migratePackagesArray(&rootObject.Members[i].Value)
73+
c.root.Format()
74+
}
75+
case 'n':
76+
// Initialize a null packages field to an empty object.
77+
rootObject.Members[i].Value.Value = &hujson.Object{
78+
AfterExtra: []byte{'\n'},
79+
}
80+
c.root.Format()
81+
}
82+
return &rootObject.Members[i]
83+
}
84+
85+
// Add a packages field to the root config object and initialize it with
86+
// an empty object.
87+
rootObject.Members = append(rootObject.Members, hujson.ObjectMember{
88+
Name: hujson.Value{
89+
Value: hujson.String("packages"),
90+
BeforeExtra: []byte{'\n'},
91+
},
92+
Value: hujson.Value{Value: &hujson.Object{}},
93+
})
94+
c.root.Format()
95+
return &rootObject.Members[len(rootObject.Members)-1]
96+
}
97+
98+
// appendPackage appends a package to the packages field.
99+
func (c *configAST) appendPackage(name, version string) {
100+
pkgs := c.packagesField(false)
101+
switch val := pkgs.Value.Value.(type) {
102+
case *hujson.Object:
103+
c.appendPackageMember(val, name, version)
104+
case *hujson.Array:
105+
c.appendPackageElement(val, joinNameVersion(name, version))
106+
default:
107+
panic("packages field must be an object or array")
108+
}
109+
110+
// Ensure the packages field is on its own line.
111+
if !slices.Contains(pkgs.Name.BeforeExtra, '\n') {
112+
pkgs.Name.BeforeExtra = append(pkgs.Name.BeforeExtra, '\n')
113+
}
114+
c.root.Format()
115+
}
116+
117+
func (c *configAST) appendPackageMember(pkgs *hujson.Object, name, version string) {
118+
i := c.memberIndex(pkgs, name)
119+
if i != -1 {
120+
return
121+
}
122+
123+
// Add a new member to the packages object with the package name and
124+
// version.
125+
pkgs.Members = append(pkgs.Members, hujson.ObjectMember{
126+
Name: hujson.Value{Value: hujson.String(name), BeforeExtra: []byte{'\n'}},
127+
Value: hujson.Value{Value: hujson.String(version)},
128+
})
129+
}
130+
131+
func (c *configAST) appendPackageElement(arr *hujson.Array, versionedName string) {
132+
var extra []byte
133+
if len(arr.Elements) > 0 {
134+
// Put each element on its own line if there
135+
// will be more than 1.
136+
extra = []byte{'\n'}
137+
}
138+
arr.Elements = append(arr.Elements, hujson.Value{
139+
BeforeExtra: extra,
140+
Value: hujson.String(versionedName),
141+
})
142+
}
143+
144+
// removePackage removes a package from the packages field.
145+
func (c *configAST) removePackage(name string) {
146+
switch val := c.packagesField(false).Value.Value.(type) {
147+
case *hujson.Object:
148+
c.removePackageMember(val, name)
149+
case *hujson.Array:
150+
c.removePackageElement(val, name)
151+
default:
152+
panic("packages field must be an object or array")
153+
}
154+
c.root.Format()
155+
}
156+
157+
func (c *configAST) removePackageMember(pkgs *hujson.Object, name string) {
158+
i := c.memberIndex(pkgs, name)
159+
if i == -1 {
160+
return
161+
}
162+
pkgs.Members = slices.Delete(pkgs.Members, i, i+1)
163+
}
164+
165+
func (c *configAST) removePackageElement(arr *hujson.Array, name string) {
166+
i := c.packageElementIndex(arr, name)
167+
if i == -1 {
168+
return
169+
}
170+
arr.Elements = slices.Delete(arr.Elements, i, i+1)
171+
}
172+
173+
// appendPlatforms appends a platform to a package's "platforms" or
174+
// "excluded_platforms" field. It automatically converts the package to an
175+
// object if it isn't already.
176+
func (c *configAST) appendPlatforms(name, fieldName string, platforms []string) {
177+
if len(platforms) == 0 {
178+
return
179+
}
180+
181+
pkgs := c.packagesField(true).Value.Value.(*hujson.Object)
182+
i := c.memberIndex(pkgs, name)
183+
if i == -1 {
184+
return
185+
}
186+
187+
// We need to ensure that the package value is a full object
188+
// (not a version string) before we can add a platform.
189+
c.convertVersionToObject(&pkgs.Members[i].Value)
190+
191+
pkgObject := pkgs.Members[i].Value.Value.(*hujson.Object)
192+
var arr *hujson.Array
193+
if i := c.memberIndex(pkgObject, fieldName); i == -1 {
194+
arr = &hujson.Array{
195+
Elements: make([]hujson.Value, 0, len(platforms)),
196+
}
197+
pkgObject.Members = append(pkgObject.Members, hujson.ObjectMember{
198+
Name: hujson.Value{
199+
Value: hujson.String(fieldName),
200+
BeforeExtra: []byte{'\n'},
201+
},
202+
Value: hujson.Value{Value: arr},
203+
})
204+
} else {
205+
arr = pkgObject.Members[i].Value.Value.(*hujson.Array)
206+
arr.Elements = slices.Grow(arr.Elements, len(platforms))
207+
}
208+
209+
for _, p := range platforms {
210+
arr.Elements = append(arr.Elements, hujson.Value{Value: hujson.String(p)})
211+
}
212+
c.root.Format()
213+
}
214+
215+
// migratePackagesArray migrates a legacy array of package versionedNames to an
216+
// object. See packagesField for details.
217+
func (c *configAST) migratePackagesArray(pkgs *hujson.Value) {
218+
arr := pkgs.Value.(*hujson.Array)
219+
obj := &hujson.Object{Members: make([]hujson.ObjectMember, len(arr.Elements))}
220+
for i, elem := range arr.Elements {
221+
name, version := parseVersionedName(elem.Value.(hujson.Literal).String())
222+
223+
// Preserve any comments above the array elements.
224+
var before []byte
225+
if comment := bytes.TrimSpace(elem.BeforeExtra); len(comment) > 0 {
226+
before = append([]byte{'\n'}, comment...)
227+
}
228+
before = append(before, '\n')
229+
230+
obj.Members[i] = hujson.ObjectMember{
231+
Name: hujson.Value{
232+
Value: hujson.String(name),
233+
BeforeExtra: before,
234+
},
235+
Value: hujson.Value{Value: hujson.String(version)},
236+
}
237+
}
238+
pkgs.Value = obj
239+
}
240+
241+
// convertVersionToObject transforms a version string into an object with the
242+
// version as a field.
243+
func (c *configAST) convertVersionToObject(pkg *hujson.Value) {
244+
if pkg.Value.Kind() == '{' {
245+
return
246+
}
247+
248+
obj := &hujson.Object{}
249+
if version, ok := pkg.Value.(hujson.Literal); ok && version.String() != "" {
250+
obj.Members = append(obj.Members, hujson.ObjectMember{
251+
Name: hujson.Value{
252+
Value: hujson.String("version"),
253+
BeforeExtra: []byte{'\n'},
254+
},
255+
Value: hujson.Value{Value: version},
256+
})
257+
}
258+
pkg.Value = obj
259+
}
260+
261+
// memberIndex returns the index of an object member.
262+
func (*configAST) memberIndex(obj *hujson.Object, name string) int {
263+
return slices.IndexFunc(obj.Members, func(m hujson.ObjectMember) bool {
264+
return m.Name.Value.(hujson.Literal).String() == name
265+
})
266+
}
267+
268+
// packageElementIndex returns the index of a package from an array of
269+
// versionedName strings.
270+
func (*configAST) packageElementIndex(arr *hujson.Array, name string) int {
271+
return slices.IndexFunc(arr.Elements, func(v hujson.Value) bool {
272+
elemName, _ := parseVersionedName(v.Value.(hujson.Literal).String())
273+
return elemName == name
274+
})
275+
}
276+
277+
func joinNameVersion(name, version string) string {
278+
if version == "" {
279+
return name
280+
}
281+
return name + "@" + version
282+
}

0 commit comments

Comments
 (0)