Skip to content

Commit c9a9255

Browse files
jirfagjirfag
authored
Speed up packages loading (#693)
Don't perform extra go env calls in go/packages. Load only needed go env vars in golangci-lint. Stay in sync by enabled analyzers in go vet: remove nilness and atomicalign analyzers, add errorsas analyzer. Don't build SSA for govet. Standalone govet runs 25% faster than before. All runs can be 5-10% faster than before. Relates: #208
1 parent 4ffe85c commit c9a9255

File tree

20 files changed

+197
-202
lines changed

20 files changed

+197
-202
lines changed

‎go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,6 @@ require (
5050
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
5151
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
5252
)
53+
54+
// https://github.com/golang/tools/pull/160
55+
replace golang.org/x/tools => github.com/golangci/tools v0.0.0-20190914130248-e9260b99c8f1

‎go.sum

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSS
106106
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
107107
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg=
108108
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
109+
github.com/golangci/tools v0.0.0-20190914130248-e9260b99c8f1 h1:8eGJVbBRoAvCh/YZq6n11s9WJkIgWKa/iTNq+R0UrNw=
110+
github.com/golangci/tools v0.0.0-20190914130248-e9260b99c8f1/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
109111
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
110112
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
111113
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -283,21 +285,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
283285
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
284286
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
285287
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
286-
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
287-
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
288-
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
289-
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
290-
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
291-
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
292-
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
293-
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
294-
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
295-
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
296-
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
297-
golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
298-
golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
299-
golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc=
300-
golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
301288
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
302289
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
303290
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

‎pkg/commands/executor.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ type Executor struct {
3131
goenv *goutil.Env
3232
fileCache *fsutils.FileCache
3333
lineCache *fsutils.LineCache
34+
debugf logutils.DebugFunc
3435
}
3536

3637
func NewExecutor(version, commit, date string) *Executor {
@@ -40,8 +41,10 @@ func NewExecutor(version, commit, date string) *Executor {
4041
commit: commit,
4142
date: date,
4243
DBManager: lintersdb.NewManager(nil),
44+
debugf: logutils.Debug("exec"),
4345
}
4446

47+
e.debugf("Starting execution...")
4548
e.log = report.NewLogWrapper(logutils.NewStderrLog(""), &e.reportData)
4649

4750
// to setup log level early we need to parse config from command line extra time to
@@ -100,10 +103,12 @@ func NewExecutor(version, commit, date string) *Executor {
100103
e.fileCache = fsutils.NewFileCache()
101104
e.lineCache = fsutils.NewLineCache(e.fileCache)
102105
e.contextLoader = lint.NewContextLoader(e.cfg, e.log.Child("loader"), e.goenv, e.lineCache, e.fileCache)
103-
106+
e.debugf("Initialized executor")
104107
return e
105108
}
106109

107110
func (e *Executor) Execute() error {
108-
return e.rootCmd.Execute()
111+
err := e.rootCmd.Execute()
112+
e.debugf("Finished execution")
113+
return err
109114
}

‎pkg/commands/help.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func printLinterConfigs(lcs []*linter.Config) {
4646
altNamesStr = fmt.Sprintf(" (%s)", strings.Join(lc.AlternativeNames, ", "))
4747
}
4848
fmt.Fprintf(logutils.StdOut, "%s%s: %s [fast: %t, auto-fix: %t]\n", color.YellowString(lc.Name()),
49-
altNamesStr, lc.Linter.Desc(), !lc.NeedsSSARepr, lc.CanAutoFix)
49+
altNamesStr, lc.Linter.Desc(), !lc.NeedsDepsTypeInfo, lc.CanAutoFix)
5050
}
5151
}
5252

‎pkg/commands/run.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ func (e *Executor) executeRun(_ *cobra.Command, args []string) {
406406
defer cancel()
407407

408408
if needTrackResources {
409-
go watchResources(ctx, trackResourcesEndCh, e.log)
409+
go watchResources(ctx, trackResourcesEndCh, e.log, e.debugf)
410410
}
411411

412412
if err := e.runAndPrint(ctx, args); err != nil {
@@ -447,11 +447,12 @@ func (e *Executor) setupExitCode(ctx context.Context) {
447447
}
448448
}
449449

450-
func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log) {
450+
func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log, debugf logutils.DebugFunc) {
451451
startedAt := time.Now()
452+
debugf("Started tracking time")
452453

453454
var rssValues []uint64
454-
ticker := time.NewTicker(100 * time.Millisecond)
455+
ticker := time.NewTicker(10 * time.Millisecond)
455456
defer ticker.Stop()
456457

457458
logEveryRecord := os.Getenv("GL_MEM_LOG_EVERY") == "1"
@@ -474,6 +475,7 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
474475
select {
475476
case <-ctx.Done():
476477
stop = true
478+
debugf("Stopped resources tracking")
477479
case <-ticker.C: // track every second
478480
}
479481

‎pkg/golinters/govet.go

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ import (
1111
"golang.org/x/tools/go/analysis/passes/asmdecl"
1212
"golang.org/x/tools/go/analysis/passes/assign"
1313
"golang.org/x/tools/go/analysis/passes/atomic"
14-
"golang.org/x/tools/go/analysis/passes/atomicalign"
1514
"golang.org/x/tools/go/analysis/passes/bools"
1615
"golang.org/x/tools/go/analysis/passes/buildtag"
1716
"golang.org/x/tools/go/analysis/passes/cgocall"
1817
"golang.org/x/tools/go/analysis/passes/composite"
1918
"golang.org/x/tools/go/analysis/passes/copylock"
19+
"golang.org/x/tools/go/analysis/passes/errorsas"
2020
"golang.org/x/tools/go/analysis/passes/httpresponse"
2121
"golang.org/x/tools/go/analysis/passes/loopclosure"
2222
"golang.org/x/tools/go/analysis/passes/lostcancel"
@@ -31,8 +31,6 @@ import (
3131
"golang.org/x/tools/go/analysis/passes/unreachable"
3232
"golang.org/x/tools/go/analysis/passes/unsafeptr"
3333
"golang.org/x/tools/go/analysis/passes/unusedresult"
34-
35-
"github.com/golangci/golangci-lint/pkg/golinters/goanalysis/passes/nilness"
3634
)
3735

3836
func NewGovet(cfg *config.GovetSettings) *goanalysis.Linter {
@@ -41,12 +39,12 @@ func NewGovet(cfg *config.GovetSettings) *goanalysis.Linter {
4139
asmdecl.Analyzer,
4240
assign.Analyzer,
4341
atomic.Analyzer,
44-
atomicalign.Analyzer,
4542
bools.Analyzer,
4643
buildtag.Analyzer,
4744
cgocall.Analyzer,
4845
composite.Analyzer,
4946
copylock.Analyzer,
47+
errorsas.Analyzer,
5048
httpresponse.Analyzer,
5149
loopclosure.Analyzer,
5250
lostcancel.Analyzer,
@@ -60,13 +58,6 @@ func NewGovet(cfg *config.GovetSettings) *goanalysis.Linter {
6058
unreachable.Analyzer,
6159
unsafeptr.Analyzer,
6260
unusedresult.Analyzer,
63-
64-
// for debugging:
65-
// findcall.Analyzer,
66-
// pkgfact.Analyzer,
67-
68-
// uses SSA:
69-
nilness.Analyzer,
7061
}
7162

7263
var settings map[string]map[string]interface{}

‎pkg/golinters/megacheck.go

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -144,15 +144,16 @@ func (MegacheckMetalinter) BuildLinterConfig(enabledChildren []string) (*linter.
144144

145145
// TODO: merge linter.Config and linter.Linter or refactor it in another way
146146
return &linter.Config{
147-
Linter: m,
148-
EnabledByDefault: false,
149-
NeedsTypeInfo: true,
150-
NeedsSSARepr: true,
151-
InPresets: []string{linter.PresetStyle, linter.PresetBugs, linter.PresetUnused},
152-
Speed: 1,
153-
AlternativeNames: nil,
154-
OriginalURL: "",
155-
ParentLinterName: "",
147+
Linter: m,
148+
EnabledByDefault: false,
149+
NeedsTypeInfo: true,
150+
NeedsDepsTypeInfo: true,
151+
NeedsSSARepr: true,
152+
InPresets: []string{linter.PresetStyle, linter.PresetBugs, linter.PresetUnused},
153+
Speed: 1,
154+
AlternativeNames: nil,
155+
OriginalURL: "",
156+
ParentLinterName: "",
156157
}, nil
157158
}
158159

‎pkg/goutil/env.go

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,21 @@ import (
55
"encoding/json"
66
"os"
77
"os/exec"
8+
"strings"
9+
"time"
810

911
"github.com/pkg/errors"
1012

1113
"github.com/golangci/golangci-lint/pkg/logutils"
1214
)
1315

16+
type EnvKey string
17+
18+
const (
19+
EnvGoCache EnvKey = "GOCACHE"
20+
EnvGoRoot EnvKey = "GOROOT"
21+
)
22+
1423
type Env struct {
1524
vars map[string]string
1625
log logutils.Log
@@ -26,24 +35,27 @@ func NewEnv(log logutils.Log) *Env {
2635
}
2736

2837
func (e *Env) Discover(ctx context.Context) error {
29-
out, err := exec.CommandContext(ctx, "go", "env", "-json").Output()
38+
startedAt := time.Now()
39+
args := []string{"env", "-json"}
40+
args = append(args, string(EnvGoCache), string(EnvGoRoot))
41+
out, err := exec.CommandContext(ctx, "go", args...).Output()
3042
if err != nil {
3143
return errors.Wrap(err, "failed to run 'go env'")
3244
}
3345

3446
if err = json.Unmarshal(out, &e.vars); err != nil {
35-
return errors.Wrap(err, "failed to parse go env json")
47+
return errors.Wrapf(err, "failed to parse 'go %s' json", strings.Join(args, " "))
3648
}
3749

38-
e.debugf("Read go env: %#v", e.vars)
50+
e.debugf("Read go env for %s: %#v", time.Since(startedAt), e.vars)
3951
return nil
4052
}
4153

42-
func (e Env) Get(k string) string {
43-
envValue := os.Getenv(k)
54+
func (e Env) Get(k EnvKey) string {
55+
envValue := os.Getenv(string(k))
4456
if envValue != "" {
4557
return envValue
4658
}
4759

48-
return e.vars[k]
60+
return e.vars[string(k)]
4961
}

‎pkg/lint/linter/config.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ type Config struct {
1313
Linter Linter
1414
EnabledByDefault bool
1515

16-
NeedsTypeInfo bool
17-
NeedsSSARepr bool
16+
NeedsTypeInfo bool
17+
NeedsDepsTypeInfo bool
18+
NeedsSSARepr bool
1819

1920
InPresets []string
2021
Speed int // more value means faster execution of linter
@@ -30,6 +31,12 @@ func (lc *Config) WithTypeInfo() *Config {
3031
return lc
3132
}
3233

34+
func (lc *Config) WithDepsTypeInfo() *Config {
35+
lc.NeedsTypeInfo = true
36+
lc.NeedsDepsTypeInfo = true
37+
return lc
38+
}
39+
3340
func (lc *Config) WithSSA() *Config {
3441
lc.NeedsTypeInfo = true
3542
lc.NeedsSSARepr = true

‎pkg/lint/lintersdb/enabled_set.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func (es EnabledSet) build(lcfg *config.Linters, enabledByDefaultLinters []*lint
5151
// It should be before --enable and --disable to be able to enable or disable specific linter.
5252
if lcfg.Fast {
5353
for name := range resultLintersSet {
54-
if es.m.GetLinterConfig(name).NeedsSSARepr {
54+
if es.m.GetLinterConfig(name).NeedsDepsTypeInfo {
5555
delete(resultLintersSet, name)
5656
}
5757
}

‎pkg/lint/lintersdb/manager.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,13 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
8585
}
8686
lcs := []*linter.Config{
8787
linter.NewConfig(golinters.NewGovet(govetCfg)).
88-
WithSSA(). // TODO: extract from the linter config and don't build SSA, just use LoadAllSyntax mode
88+
WithDepsTypeInfo().
8989
WithPresets(linter.PresetBugs).
9090
WithSpeed(4).
9191
WithAlternativeNames("vet", "vetshadow").
9292
WithURL("https://golang.org/cmd/vet/"),
9393
linter.NewConfig(golinters.NewBodyclose()).
94+
WithDepsTypeInfo().
9495
WithSSA().
9596
WithPresets(linter.PresetPerformance, linter.PresetBugs).
9697
WithSpeed(4).
@@ -106,21 +107,25 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
106107
WithURL("https://github.com/golang/lint"),
107108

108109
linter.NewConfig(golinters.NewStaticcheck()).
110+
WithDepsTypeInfo().
109111
WithSSA().
110112
WithPresets(linter.PresetBugs).
111113
WithSpeed(2).
112114
WithURL("https://staticcheck.io/"),
113115
linter.NewConfig(golinters.NewUnused()).
116+
WithDepsTypeInfo().
114117
WithSSA().
115118
WithPresets(linter.PresetUnused).
116119
WithSpeed(5).
117120
WithURL("https://github.com/dominikh/go-tools/tree/master/cmd/unused"),
118121
linter.NewConfig(golinters.NewGosimple()).
122+
WithDepsTypeInfo().
119123
WithSSA().
120124
WithPresets(linter.PresetStyle).
121125
WithSpeed(5).
122126
WithURL("https://github.com/dominikh/go-tools/tree/master/cmd/gosimple"),
123127
linter.NewConfig(golinters.NewStylecheck()).
128+
WithDepsTypeInfo().
124129
WithSSA().
125130
WithPresets(linter.PresetStyle).
126131
WithSpeed(5).
@@ -143,6 +148,7 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
143148
WithSpeed(10).
144149
WithURL("https://github.com/opennota/check"),
145150
linter.NewConfig(golinters.Interfacer{}).
151+
WithDepsTypeInfo().
146152
WithSSA().
147153
WithPresets(linter.PresetStyle).
148154
WithSpeed(6).
@@ -211,6 +217,7 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
211217
linter.NewConfig(golinters.Unparam{}).
212218
WithPresets(linter.PresetUnused).
213219
WithSpeed(3).
220+
WithDepsTypeInfo().
214221
WithSSA().
215222
WithURL("https://github.com/mvdan/unparam"),
216223
linter.NewConfig(golinters.Nakedret{}).

‎pkg/lint/load.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func NewContextLoader(cfg *config.Config, log logutils.Log, goenv *goutil.Env,
5353
func (cl ContextLoader) prepareBuildContext() {
5454
// Set GOROOT to have working cross-compilation: cross-compiled binaries
5555
// have invalid GOROOT. XXX: can't use runtime.GOROOT().
56-
goroot := cl.goenv.Get("GOROOT")
56+
goroot := cl.goenv.Get(goutil.EnvGoRoot)
5757
if goroot == "" {
5858
return
5959
}
@@ -149,7 +149,7 @@ func (cl ContextLoader) findLoadMode(linters []*linter.Config) packages.LoadMode
149149
if lc.NeedsTypeInfo {
150150
loadMode |= packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedTypesInfo | packages.NeedSyntax
151151
}
152-
if lc.NeedsSSARepr {
152+
if lc.NeedsDepsTypeInfo {
153153
loadMode |= packages.NeedDeps
154154
}
155155
}
@@ -349,6 +349,15 @@ func (cl ContextLoader) filterDuplicatePackages(pkgs []*packages.Package) []*pac
349349
return retPkgs
350350
}
351351

352+
func needSSA(linters []*linter.Config) bool {
353+
for _, lc := range linters {
354+
if lc.NeedsSSARepr {
355+
return true
356+
}
357+
}
358+
return false
359+
}
360+
352361
//nolint:gocyclo
353362
func (cl ContextLoader) Load(ctx context.Context, linters []*linter.Config) (*linter.Context, error) {
354363
loadMode := cl.findLoadMode(linters)
@@ -369,7 +378,7 @@ func (cl ContextLoader) Load(ctx context.Context, linters []*linter.Config) (*li
369378
}
370379

371380
var ssaProg *ssa.Program
372-
if loadMode&packages.NeedDeps != 0 {
381+
if needSSA(linters) {
373382
ssaProg = cl.buildSSAProgram(deduplicatedPkgs)
374383
}
375384

‎pkg/logutils/logutils.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ func Debug(tag string) DebugFunc {
3030
return nopDebugf
3131
}
3232

33+
logger := NewStderrLog(tag)
34+
logger.SetLevel(LogLevelDebug)
35+
3336
return func(format string, args ...interface{}) {
34-
logger := NewStderrLog(tag)
35-
logger.SetLevel(LogLevelDebug)
3637
logger.Debugf(format, args...)
3738
}
3839
}

0 commit comments

Comments
 (0)