Skip to content

Commit c0bc050

Browse files
committed
use filecache
1 parent 5bb257f commit c0bc050

File tree

5 files changed

+106
-72
lines changed

5 files changed

+106
-72
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ require (
3939
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
4040
github.com/wk8/go-ordered-map/v2 v2.1.8
4141
github.com/zealic/go2node v0.1.0
42-
go.jetpack.io/pkg v0.0.0-20231012130632-77dcd111db2e
42+
go.jetpack.io/pkg v0.0.0-20231019173032-13f60a9e32b8
4343
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
4444
golang.org/x/mod v0.12.0
4545
golang.org/x/sync v0.3.0

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,8 +339,8 @@ github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBU
339339
github.com/zaffka/mongodb-boltdb-mock v0.0.0-20221014194232-b4bb03fbe3a0/go.mod h1:GsDD1qsG+86MeeCG7ndi6Ei3iGthKL3wQ7PTFigDfNY=
340340
github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA=
341341
github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg=
342-
go.jetpack.io/pkg v0.0.0-20231012130632-77dcd111db2e h1:GtqFrE5uUf6rXFk3zaIqIfLfykrqrB8gp6bDmOfH+2s=
343-
go.jetpack.io/pkg v0.0.0-20231012130632-77dcd111db2e/go.mod h1:m49CAcLbpZttEbzoEWC1SKD+/z5mEiFdw0dQxY6AM5Q=
342+
go.jetpack.io/pkg v0.0.0-20231019173032-13f60a9e32b8 h1:xzAAUVfg9uqyJhZVc+ZDTLHtFlMhj/5St5JnVfDxbpk=
343+
go.jetpack.io/pkg v0.0.0-20231019173032-13f60a9e32b8/go.mod h1:m49CAcLbpZttEbzoEWC1SKD+/z5mEiFdw0dQxY6AM5Q=
344344
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
345345
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
346346
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

internal/devpkg/package.go

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ func (p *Package) FullPackageAttributePath() (string, error) {
270270
}
271271

272272
// NormalizedPackageAttributePath returns an attribute path normalized by nix
273-
// search. This is useful for comparing different attribute paths that may
273+
// lookupNixInfo. This is useful for comparing different attribute paths that may
274274
// point to the same package. Note, it may be an expensive call.
275275
func (p *Package) NormalizedPackageAttributePath() (string, error) {
276276
if p.normalizedPackageAttributePathCache != "" {
@@ -284,24 +284,7 @@ func (p *Package) NormalizedPackageAttributePath() (string, error) {
284284
return p.normalizedPackageAttributePathCache, nil
285285
}
286286

287-
// search attempts to find the package in the local nixpkgs. It may be an expensive
288-
// call, if it is has not been cached.
289-
func (p *Package) search(query string) (map[string]*nix.Info, error) {
290-
if p.IsDevboxPackage() {
291-
if strings.HasPrefix(query, "runx:") {
292-
// TODO implement runx search
293-
return map[string]*nix.Info{}, nil
294-
}
295-
// This will be slow if its the first time on the user's machine that this
296-
// query is running. Otherwise, it will be cached and fast.
297-
return nix.SearchNixpkgsAttribute(query)
298-
}
299-
300-
// fallback to the slow but generalized nix.Search
301-
return nix.Search(query)
302-
}
303-
304-
// normalizePackageAttributePath calls nix search to find the normalized attribute
287+
// normalizePackageAttributePath calls nix lookupNixInfo to find the normalized attribute
305288
// path. It may be an expensive call (~100ms).
306289
func (p *Package) normalizePackageAttributePath() (string, error) {
307290
var query string
@@ -319,11 +302,24 @@ func (p *Package) normalizePackageAttributePath() (string, error) {
319302
query = p.String()
320303
}
321304

322-
// We prefer search over just trying to parse the package's "URL" because search will
323-
// guarantee that the package exists for the current system.
324-
infos, err := p.search(query)
325-
if err != nil {
326-
return "", err
305+
// We prefer nix.Search over just trying to parse the package's "URL" because
306+
// nix.Search will guarantee that the package exists for the current system.
307+
var infos map[string]*nix.Info
308+
var err error
309+
if p.IsDevboxPackage() && !p.IsRunX() {
310+
// Perf optimization: For queries of the form nixpkgs/<commit>#foo, we can
311+
// use a nix.Search cache.
312+
//
313+
// This will be slow if its the first time on the user's machine that this
314+
// query is running. Otherwise, it will be cached and fast.
315+
if infos, err = nix.SearchNixpkgsAttribute(query); err != nil {
316+
return "", err
317+
}
318+
} else {
319+
// fallback to the slow but generalized nix.Search
320+
if infos, err = nix.Search(query); err != nil {
321+
return "", err
322+
}
327323
}
328324

329325
if len(infos) == 1 {

internal/nix/search.go

Lines changed: 57 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
package nix
22

33
import (
4-
"bytes"
54
"encoding/json"
65
"fmt"
76
"os"
87
"os/exec"
9-
"path/filepath"
8+
"regexp"
9+
"strings"
10+
"time"
1011

1112
"github.com/pkg/errors"
1213
"go.jetpack.io/devbox/internal/debug"
1314
"go.jetpack.io/devbox/internal/xdg"
15+
"go.jetpack.io/pkg/filecache"
1416
)
1517

1618
var (
@@ -32,6 +34,11 @@ func (i *Info) String() string {
3234
}
3335

3436
func Search(url string) (map[string]*Info, error) {
37+
if strings.HasPrefix(url, "runx:") {
38+
// TODO implement runx search. Also, move this check outside this function: nix package
39+
// should not be handling runx logic.
40+
return map[string]*Info{}, nil
41+
}
3542
return searchSystem(url, "" /* system */)
3643
}
3744

@@ -113,11 +120,15 @@ type searchSystemCache struct {
113120

114121
const (
115122
// searchSystemCacheSubDir is a sub-directory of the XDG cache directory
116-
searchSystemCacheSubDir = "devbox/nix"
123+
searchSystemCacheSubDir = "devbox/nix-search"
117124
searchSystemCacheFileName = "search-system-cache.json"
118125
)
119126

120-
var cache = searchSystemCache{}
127+
type CachedSearchResult struct {
128+
Results map[string]*Info `json:"results"`
129+
// Query is added easier to grep for debuggability
130+
Query string `json:"query"`
131+
}
121132

122133
// SearchNixpkgsAttribute is a wrapper around searchSystem that caches results.
123134
// NOTE: we should be very conservative in where we use this function. `nix search`
@@ -126,63 +137,64 @@ var cache = searchSystemCache{}
126137
// once `nix search` returns a valid result, it will always be the very same result.
127138
// Hence we can cache it locally and answer future queries fast, by not calling `nix search`.
128139
func SearchNixpkgsAttribute(query string) (map[string]*Info, error) {
129-
if cache.QueryToInfo == nil {
130-
contents, err := readSearchSystemCacheFile()
131-
if err != nil {
140+
key := cacheKey(query)
141+
142+
// Check if the query was already cached, and return the result if so
143+
cache := filecache.New("devbox/nix", filecache.WithCacheDir(xdg.CacheSubpath("")))
144+
if cachedResults, err := cache.Get(key); err == nil {
145+
var results map[string]*Info
146+
if err := json.Unmarshal(cachedResults, &results); err != nil {
132147
return nil, err
133148
}
134-
cache.QueryToInfo = contents
135-
}
136-
137-
if result := cache.QueryToInfo[query]; result != nil {
138-
return result, nil
149+
return results, nil
150+
} else if !filecacheNeedsUpdate(err) {
151+
return nil, err // genuine error
139152
}
140153

141-
info, err := searchSystem(query, "" /*system*/)
154+
// If not cached, or an update is needed, then call searchSystem
155+
infos, err := searchSystem(query, "" /*system*/)
142156
if err != nil {
143157
return nil, err
144158
}
145159

146-
cache.QueryToInfo[query] = info
147-
if err := writeSearchSystemCacheFile(cache.QueryToInfo); err != nil {
148-
return nil, err
149-
}
150-
151-
return info, nil
152-
}
153-
154-
func readSearchSystemCacheFile() (map[string]map[string]*Info, error) {
155-
contents, err := os.ReadFile(xdg.CacheSubpath(filepath.Join(searchSystemCacheSubDir, searchSystemCacheFileName)))
160+
// Save the results to the cache
161+
marshalled, err := json.Marshal(infos)
156162
if err != nil {
157-
if os.IsNotExist(err) {
158-
// If the file doesn't exist, return an empty map. This will hopefully be filled and written to disk later.
159-
return make(map[string]map[string]*Info), nil
160-
}
161163
return nil, err
162164
}
163-
164-
var result map[string]map[string]*Info
165-
if err := json.Unmarshal(contents, &result); err != nil {
165+
// TODO savil: add a SetForever API that does not expire. Time based expiration is not needed here
166+
// because we're caching results that are guaranteed to be stable.
167+
// TODO savil: Make filecache.cache a public struct so it can be passed into other functions
168+
const oneYear = 12 * 30 * 24 * time.Hour
169+
if err := cache.Set(key, marshalled, oneYear); err != nil {
166170
return nil, err
167171
}
168-
return result, nil
172+
173+
return infos, nil
169174
}
170175

171-
func writeSearchSystemCacheFile(contents map[string]map[string]*Info) error {
172-
// Print as a human-readable JSON file i.e. use nice indentation and newlines.
173-
buf := bytes.Buffer{}
174-
enc := json.NewEncoder(&buf)
175-
enc.SetIndent("", " ")
176-
err := enc.Encode(contents)
177-
if err != nil {
178-
return err
179-
}
176+
// read as: filecache.NeedsUpdate(err)
177+
// TODO savil: this should be implemented in the filecache package
178+
func filecacheNeedsUpdate(err error) bool {
179+
return err == filecache.NotFound || err == filecache.Expired
180+
}
180181

181-
dir := xdg.CacheSubpath(searchSystemCacheSubDir)
182-
if err := os.MkdirAll(dir, 0o755); err != nil {
183-
return err
182+
// cacheKey sanitizes the search query to be a valid unix filename.
183+
// This cache key is used as the filename to store the cache value, and having a
184+
// representation of the query is important for debuggability.
185+
func cacheKey(input string) string {
186+
// Replace disallowed characters with underscores.
187+
re := regexp.MustCompile(`[:/#+]`)
188+
sanitized := re.ReplaceAllString(input, "_")
189+
190+
// Remove any remaining invalid characters.
191+
sanitized = regexp.MustCompile(`[^\w\.-]`).ReplaceAllString(sanitized, "")
192+
193+
// Ensure the filename doesn't exceed the maximum length.
194+
const maxLen = 255
195+
if len(sanitized) > maxLen {
196+
sanitized = sanitized[:maxLen]
184197
}
185198

186-
path := filepath.Join(dir, searchSystemCacheFileName)
187-
return os.WriteFile(path, buf.Bytes(), 0o644)
199+
return sanitized
188200
}

internal/nix/search_test.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package nix
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestSearchCacheKey(t *testing.T) {
8+
testCases := []struct {
9+
in string
10+
out string
11+
}{
12+
{
13+
"github:NixOS/nixpkgs/8670e496ffd093b60e74e7fa53526aa5920d09eb#go_1_19",
14+
"github_NixOS_nixpkgs_8670e496ffd093b60e74e7fa53526aa5920d09eb_go_1_19",
15+
},
16+
}
17+
18+
for _, testCase := range testCases {
19+
t.Run(testCase.out, func(t *testing.T) {
20+
out := cacheKey(testCase.in)
21+
if out != testCase.out {
22+
t.Errorf("got %s, want %s", out, testCase.out)
23+
}
24+
})
25+
}
26+
}

0 commit comments

Comments
 (0)