Skip to content

Commit df4f676

Browse files
authored
reduce 1.5x memory usage on large repos on repeated runs (#764)
Get rid of AST cache: load AST when needed. Optimize memory allocations for go/analysis actions. Relates: #337
1 parent ea417ff commit df4f676

File tree

15 files changed

+264
-347
lines changed

15 files changed

+264
-347
lines changed

pkg/commands/run.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -290,8 +290,8 @@ func (e *Executor) runAnalysis(ctx context.Context, args []string) ([]result.Iss
290290
}
291291
lintCtx.Log = e.log.Child("linters context")
292292

293-
runner, err := lint.NewRunner(lintCtx.ASTCache, e.cfg, e.log.Child("runner"),
294-
e.goenv, e.lineCache, e.DBManager)
293+
runner, err := lint.NewRunner(e.cfg, e.log.Child("runner"),
294+
e.goenv, e.lineCache, e.DBManager, lintCtx.Packages)
295295
if err != nil {
296296
return nil, err
297297
}

pkg/golinters/goanalysis/linter.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ func buildIssuesFromErrorsForTypecheckMode(errs []error, lintCtx *linter.Context
189189
if !ok {
190190
return nil, err
191191
}
192-
for _, err := range libpackages.ExtractErrors(itErr.Pkg, lintCtx.ASTCache) {
192+
for _, err := range libpackages.ExtractErrors(itErr.Pkg) {
193193
i, perr := parseError(err)
194194
if perr != nil { // failed to parse
195195
if uniqReportedIssues[err.Msg] {

pkg/golinters/goanalysis/runner.go

Lines changed: 127 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,113 @@ func (r *runner) run(analyzers []*analysis.Analyzer, initialPackages []*packages
108108
return extractDiagnostics(roots)
109109
}
110110

111+
type actKey struct {
112+
*analysis.Analyzer
113+
*packages.Package
114+
}
115+
116+
func (r *runner) markAllActions(a *analysis.Analyzer, pkg *packages.Package, markedActions map[actKey]struct{}) {
117+
k := actKey{a, pkg}
118+
if _, ok := markedActions[k]; ok {
119+
return
120+
}
121+
122+
for _, req := range a.Requires {
123+
r.markAllActions(req, pkg, markedActions)
124+
}
125+
126+
if len(a.FactTypes) != 0 {
127+
for path := range pkg.Imports {
128+
r.markAllActions(a, pkg.Imports[path], markedActions)
129+
}
130+
}
131+
132+
markedActions[k] = struct{}{}
133+
}
134+
135+
func (r *runner) makeAction(a *analysis.Analyzer, pkg *packages.Package,
136+
initialPkgs map[*packages.Package]bool, actions map[actKey]*action, actAlloc *actionAllocator) *action {
137+
k := actKey{a, pkg}
138+
act, ok := actions[k]
139+
if ok {
140+
return act
141+
}
142+
143+
act = actAlloc.alloc()
144+
act.a = a
145+
act.pkg = pkg
146+
act.log = r.log
147+
act.prefix = r.prefix
148+
act.pkgCache = r.pkgCache
149+
act.isInitialPkg = initialPkgs[pkg]
150+
act.needAnalyzeSource = initialPkgs[pkg]
151+
act.analysisDoneCh = make(chan struct{})
152+
153+
depsCount := len(a.Requires)
154+
if len(a.FactTypes) > 0 {
155+
depsCount += len(pkg.Imports)
156+
}
157+
act.deps = make([]*action, 0, depsCount)
158+
159+
// Add a dependency on each required analyzers.
160+
for _, req := range a.Requires {
161+
act.deps = append(act.deps, r.makeAction(req, pkg, initialPkgs, actions, actAlloc))
162+
}
163+
164+
r.buildActionFactDeps(act, a, pkg, initialPkgs, actions, actAlloc)
165+
166+
actions[k] = act
167+
return act
168+
}
169+
170+
func (r *runner) buildActionFactDeps(act *action, a *analysis.Analyzer, pkg *packages.Package,
171+
initialPkgs map[*packages.Package]bool, actions map[actKey]*action, actAlloc *actionAllocator) {
172+
// An analysis that consumes/produces facts
173+
// must run on the package's dependencies too.
174+
if len(a.FactTypes) == 0 {
175+
return
176+
}
177+
178+
act.objectFacts = make(map[objectFactKey]analysis.Fact)
179+
act.packageFacts = make(map[packageFactKey]analysis.Fact)
180+
181+
paths := make([]string, 0, len(pkg.Imports))
182+
for path := range pkg.Imports {
183+
paths = append(paths, path)
184+
}
185+
sort.Strings(paths) // for determinism
186+
for _, path := range paths {
187+
dep := r.makeAction(a, pkg.Imports[path], initialPkgs, actions, actAlloc)
188+
act.deps = append(act.deps, dep)
189+
}
190+
191+
// Need to register fact types for pkgcache proper gob encoding.
192+
for _, f := range a.FactTypes {
193+
gob.Register(f)
194+
}
195+
}
196+
197+
type actionAllocator struct {
198+
allocatedActions []action
199+
nextFreeIndex int
200+
}
201+
202+
func newActionAllocator(maxCount int) *actionAllocator {
203+
return &actionAllocator{
204+
allocatedActions: make([]action, maxCount),
205+
nextFreeIndex: 0,
206+
}
207+
}
208+
209+
func (actAlloc *actionAllocator) alloc() *action {
210+
if actAlloc.nextFreeIndex == len(actAlloc.allocatedActions) {
211+
panic(fmt.Sprintf("Made too many allocations of actions: %d allowed", len(actAlloc.allocatedActions)))
212+
}
213+
act := &actAlloc.allocatedActions[actAlloc.nextFreeIndex]
214+
actAlloc.nextFreeIndex++
215+
return act
216+
}
217+
111218
//nolint:gocritic
112219
func (r *runner) prepareAnalysis(pkgs []*packages.Package,
113220
analyzers []*analysis.Analyzer) (map[*packages.Package]bool, []*action, []*action) {
@@ -116,70 +223,30 @@ func (r *runner) prepareAnalysis(pkgs []*packages.Package,
116223
// Each graph node (action) is one unit of analysis.
117224
// Edges express package-to-package (vertical) dependencies,
118225
// and analysis-to-analysis (horizontal) dependencies.
119-
type key struct {
120-
*analysis.Analyzer
121-
*packages.Package
122-
}
123-
actions := make(map[key]*action)
124226

125-
initialPkgs := map[*packages.Package]bool{}
126-
for _, pkg := range pkgs {
127-
initialPkgs[pkg] = true
227+
// This place is memory-intensive: e.g. Istio project has 120k total actions.
228+
// Therefore optimize it carefully.
229+
markedActions := make(map[actKey]struct{}, len(analyzers)*len(pkgs))
230+
for _, a := range analyzers {
231+
for _, pkg := range pkgs {
232+
r.markAllActions(a, pkg, markedActions)
233+
}
128234
}
235+
totalActionsCount := len(markedActions)
129236

130-
var mkAction func(a *analysis.Analyzer, pkg *packages.Package) *action
131-
mkAction = func(a *analysis.Analyzer, pkg *packages.Package) *action {
132-
k := key{a, pkg}
133-
act, ok := actions[k]
134-
if !ok {
135-
act = &action{
136-
a: a,
137-
pkg: pkg,
138-
log: r.log,
139-
prefix: r.prefix,
140-
pkgCache: r.pkgCache,
141-
isInitialPkg: initialPkgs[pkg],
142-
needAnalyzeSource: initialPkgs[pkg],
143-
analysisDoneCh: make(chan struct{}),
144-
objectFacts: make(map[objectFactKey]analysis.Fact),
145-
packageFacts: make(map[packageFactKey]analysis.Fact),
146-
loadMode: r.loadMode,
147-
}
148-
149-
// Add a dependency on each required analyzers.
150-
for _, req := range a.Requires {
151-
act.deps = append(act.deps, mkAction(req, pkg))
152-
}
237+
actions := make(map[actKey]*action, totalActionsCount)
238+
actAlloc := newActionAllocator(totalActionsCount)
153239

154-
// An analysis that consumes/produces facts
155-
// must run on the package's dependencies too.
156-
if len(a.FactTypes) > 0 {
157-
paths := make([]string, 0, len(pkg.Imports))
158-
for path := range pkg.Imports {
159-
paths = append(paths, path)
160-
}
161-
sort.Strings(paths) // for determinism
162-
for _, path := range paths {
163-
dep := mkAction(a, pkg.Imports[path])
164-
act.deps = append(act.deps, dep)
165-
}
166-
167-
// Need to register fact types for pkgcache proper gob encoding.
168-
for _, f := range a.FactTypes {
169-
gob.Register(f)
170-
}
171-
}
172-
173-
actions[k] = act
174-
}
175-
return act
240+
initialPkgs := make(map[*packages.Package]bool, len(pkgs))
241+
for _, pkg := range pkgs {
242+
initialPkgs[pkg] = true
176243
}
177244

178245
// Build nodes for initial packages.
179-
var roots []*action
246+
roots := make([]*action, 0, len(pkgs)*len(analyzers))
180247
for _, a := range analyzers {
181248
for _, pkg := range pkgs {
182-
root := mkAction(a, pkg)
249+
root := r.makeAction(a, pkg, initialPkgs, actions, actAlloc)
183250
root.isroot = true
184251
roots = append(roots, root)
185252
}
@@ -190,6 +257,8 @@ func (r *runner) prepareAnalysis(pkgs []*packages.Package,
190257
allActions = append(allActions, act)
191258
}
192259

260+
debugf("Built %d actions", len(actions))
261+
193262
return initialPkgs, allActions, roots
194263
}
195264

@@ -334,9 +403,6 @@ type action struct {
334403
a *analysis.Analyzer
335404
pkg *packages.Package
336405
pass *analysis.Pass
337-
isroot bool
338-
isInitialPkg bool
339-
needAnalyzeSource bool
340406
deps []*action
341407
objectFacts map[objectFactKey]analysis.Fact
342408
packageFacts map[packageFactKey]analysis.Fact
@@ -349,7 +415,9 @@ type action struct {
349415
analysisDoneCh chan struct{}
350416
loadCachedFactsDone bool
351417
loadCachedFactsOk bool
352-
loadMode LoadMode
418+
isroot bool
419+
isInitialPkg bool
420+
needAnalyzeSource bool
353421
}
354422

355423
type objectFactKey struct {

0 commit comments

Comments
 (0)