Skip to content

Commit adb6be7

Browse files
jirfagjirfag
committed
Fix #72: match more autogenerated files patterns.
We skip all issues from autogenerated files. Also reuse AST parsing for nolint and autogenerated exclude processors: decrease processing time on golang source code from 3s to 800ms.
1 parent 46088de commit adb6be7

File tree

11 files changed

+232
-87
lines changed

11 files changed

+232
-87
lines changed

Diff for: ‎pkg/commands/run.go

+3-7
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package commands
33
import (
44
"context"
55
"fmt"
6-
"go/token"
76
"io/ioutil"
87
"log"
98
"os"
@@ -192,10 +191,6 @@ func (e *Executor) runAnalysis(ctx context.Context, args []string) (<-chan resul
192191
if len(excludePatterns) != 0 {
193192
excludeTotalPattern = fmt.Sprintf("(%s)", strings.Join(excludePatterns, "|"))
194193
}
195-
fset := token.NewFileSet()
196-
if lintCtx.Program != nil {
197-
fset = lintCtx.Program.Fset
198-
}
199194

200195
skipFilesProcessor, err := processors.NewSkipFiles(e.cfg.Run.SkipFiles)
201196
if err != nil {
@@ -204,12 +199,13 @@ func (e *Executor) runAnalysis(ctx context.Context, args []string) (<-chan resul
204199

205200
runner := lint.SimpleRunner{
206201
Processors: []processors.Processor{
207-
processors.NewPathPrettifier(), // must be before diff processor at least
202+
processors.NewPathPrettifier(), // must be before diff, nolint and exclude autogenerated processor at least
208203
processors.NewCgo(),
209204
skipFilesProcessor,
210205

206+
processors.NewAutogeneratedExclude(lintCtx.ASTCache),
211207
processors.NewExclude(excludeTotalPattern),
212-
processors.NewNolint(fset),
208+
processors.NewNolint(lintCtx.ASTCache),
213209

214210
processors.NewUniqByLine(),
215211
processors.NewDiff(e.cfg.Issues.Diff, e.cfg.Issues.DiffFromRevision, e.cfg.Issues.DiffPatchFilePath),

Diff for: ‎pkg/lint/astcache/astcache.go

+32-8
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,27 @@ type Cache struct {
2424
s []*File
2525
}
2626

27+
func NewCache() *Cache {
28+
return &Cache{
29+
m: map[string]*File{},
30+
}
31+
}
32+
2733
func (c Cache) Get(filename string) *File {
2834
return c.m[filepath.Clean(filename)]
2935
}
3036

37+
func (c Cache) GetOrParse(filename string) *File {
38+
f := c.m[filename]
39+
if f != nil {
40+
return f
41+
}
42+
43+
logrus.Infof("Parse AST for file %s on demand", filename)
44+
c.parseFile(filename, nil)
45+
return c.m[filename]
46+
}
47+
3148
func (c Cache) GetAllValidFiles() []*File {
3249
return c.s
3350
}
@@ -79,6 +96,20 @@ func LoadFromProgram(prog *loader.Program) (*Cache, error) {
7996
return c, nil
8097
}
8198

99+
func (c *Cache) parseFile(filePath string, fset *token.FileSet) {
100+
if fset == nil {
101+
fset = token.NewFileSet()
102+
}
103+
104+
f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) // comments needed by e.g. golint
105+
c.m[filePath] = &File{
106+
F: f,
107+
Fset: fset,
108+
Err: err,
109+
Name: filePath,
110+
}
111+
}
112+
82113
func LoadFromFiles(files []string) (*Cache, error) {
83114
c := &Cache{
84115
m: map[string]*File{},
@@ -87,14 +118,7 @@ func LoadFromFiles(files []string) (*Cache, error) {
87118
fset := token.NewFileSet()
88119
for _, filePath := range files {
89120
filePath = filepath.Clean(filePath)
90-
91-
f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) // comments needed by e.g. golint
92-
c.m[filePath] = &File{
93-
F: f,
94-
Fset: fset,
95-
Err: err,
96-
Name: filePath,
97-
}
121+
c.parseFile(filePath, fset)
98122
}
99123

100124
c.prepareValidFiles()

Diff for: ‎pkg/result/processors/autogenerated_exclude.go

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package processors
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/golangci/golangci-lint/pkg/lint/astcache"
8+
"github.com/golangci/golangci-lint/pkg/result"
9+
)
10+
11+
type ageFileSummary struct {
12+
isGenerated bool
13+
}
14+
15+
type ageFileSummaryCache map[string]*ageFileSummary
16+
17+
type AutogeneratedExclude struct {
18+
fileSummaryCache ageFileSummaryCache
19+
astCache *astcache.Cache
20+
}
21+
22+
func NewAutogeneratedExclude(astCache *astcache.Cache) *AutogeneratedExclude {
23+
return &AutogeneratedExclude{
24+
fileSummaryCache: ageFileSummaryCache{},
25+
astCache: astCache,
26+
}
27+
}
28+
29+
var _ Processor = &AutogeneratedExclude{}
30+
31+
func (p AutogeneratedExclude) Name() string {
32+
return "autogenerated_exclude"
33+
}
34+
35+
func (p *AutogeneratedExclude) Process(issues []result.Issue) ([]result.Issue, error) {
36+
return filterIssuesErr(issues, p.shouldPassIssue)
37+
}
38+
39+
func (p *AutogeneratedExclude) shouldPassIssue(i *result.Issue) (bool, error) {
40+
fs, err := p.getOrCreateFileSummary(i)
41+
if err != nil {
42+
return false, err
43+
}
44+
45+
// don't report issues for autogenerated files
46+
return !fs.isGenerated, nil
47+
}
48+
49+
// isGenerated reports whether the source file is generated code.
50+
// Using a bit laxer rules than https://golang.org/s/generatedcode to
51+
// match more generated code. See #48 and #72.
52+
func isGeneratedFileByComment(doc string) bool {
53+
const (
54+
genCodeGenerated = "code generated"
55+
genDoNotEdit = "do not edit"
56+
genAutoFile = "autogenerated file" // easyjson
57+
)
58+
59+
markers := []string{genCodeGenerated, genDoNotEdit, genAutoFile}
60+
doc = strings.ToLower(doc)
61+
for _, marker := range markers {
62+
if strings.Contains(doc, marker) {
63+
return true
64+
}
65+
}
66+
67+
return false
68+
}
69+
70+
func (p *AutogeneratedExclude) getOrCreateFileSummary(i *result.Issue) (*ageFileSummary, error) {
71+
fs := p.fileSummaryCache[i.FilePath()]
72+
if fs != nil {
73+
return fs, nil
74+
}
75+
76+
fs = &ageFileSummary{}
77+
p.fileSummaryCache[i.FilePath()] = fs
78+
79+
f := p.astCache.GetOrParse(i.FilePath())
80+
if f.Err != nil {
81+
return nil, fmt.Errorf("can't parse file %s: %s", i.FilePath(), f.Err)
82+
}
83+
84+
fs.isGenerated = isGeneratedFileByComment(f.F.Doc.Text())
85+
return fs, nil
86+
}
87+
88+
func (p AutogeneratedExclude) Finish() {}

Diff for: ‎pkg/result/processors/autogenerated_exclude_test.go

+88
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎pkg/result/processors/nolint.go

+10-51
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,13 @@
11
package processors
22

33
import (
4-
"bufio"
5-
"bytes"
64
"fmt"
75
"go/ast"
8-
"go/parser"
96
"go/token"
10-
"io/ioutil"
117
"sort"
128
"strings"
139

10+
"github.com/golangci/golangci-lint/pkg/lint/astcache"
1411
"github.com/golangci/golangci-lint/pkg/result"
1512
)
1613

@@ -44,20 +41,19 @@ func (i *ignoredRange) doesMatch(issue *result.Issue) bool {
4441

4542
type fileData struct {
4643
ignoredRanges []ignoredRange
47-
isGenerated bool
4844
}
4945

5046
type filesCache map[string]*fileData
5147

5248
type Nolint struct {
53-
fset *token.FileSet
54-
cache filesCache
49+
cache filesCache
50+
astCache *astcache.Cache
5551
}
5652

57-
func NewNolint(fset *token.FileSet) *Nolint {
53+
func NewNolint(astCache *astcache.Cache) *Nolint {
5854
return &Nolint{
59-
fset: fset,
60-
cache: filesCache{},
55+
cache: filesCache{},
56+
astCache: astCache,
6157
}
6258
}
6359

@@ -71,29 +67,6 @@ func (p *Nolint) Process(issues []result.Issue) ([]result.Issue, error) {
7167
return filterIssuesErr(issues, p.shouldPassIssue)
7268
}
7369

74-
var (
75-
genHdr = []byte("// Code generated")
76-
genFtr = []byte("DO NOT EDIT")
77-
)
78-
79-
// isGenerated reports whether the source file is generated code.
80-
// Using a bit laxer rules than https://golang.org/s/generatedcode to
81-
// match more generated code.
82-
func isGenerated(src []byte) bool {
83-
sc := bufio.NewScanner(bytes.NewReader(src))
84-
var hdr, ftr bool
85-
for sc.Scan() {
86-
b := sc.Bytes()
87-
if bytes.HasPrefix(b, genHdr) {
88-
hdr = true
89-
}
90-
if bytes.Contains(b, genFtr) {
91-
ftr = true
92-
}
93-
}
94-
return hdr && ftr
95-
}
96-
9770
func (p *Nolint) getOrCreateFileData(i *result.Issue) (*fileData, error) {
9871
fd := p.cache[i.FilePath()]
9972
if fd != nil {
@@ -103,22 +76,12 @@ func (p *Nolint) getOrCreateFileData(i *result.Issue) (*fileData, error) {
10376
fd = &fileData{}
10477
p.cache[i.FilePath()] = fd
10578

106-
src, err := ioutil.ReadFile(i.FilePath())
107-
if err != nil {
108-
return nil, fmt.Errorf("can't read file %s: %s", i.FilePath(), err)
109-
}
110-
111-
fd.isGenerated = isGenerated(src)
112-
if fd.isGenerated { // don't report issues for autogenerated files
113-
return fd, nil
114-
}
115-
116-
file, err := parser.ParseFile(p.fset, i.FilePath(), src, parser.ParseComments)
117-
if err != nil {
118-
return nil, fmt.Errorf("can't parse file %s", i.FilePath())
79+
file := p.astCache.GetOrParse(i.FilePath())
80+
if file.Err != nil {
81+
return nil, fmt.Errorf("can't parse file %s: %s", i.FilePath(), file.Err)
11982
}
12083

121-
fd.ignoredRanges = buildIgnoredRangesForFile(file, p.fset)
84+
fd.ignoredRanges = buildIgnoredRangesForFile(file.F, file.Fset)
12285
return fd, nil
12386
}
12487

@@ -145,10 +108,6 @@ func (p *Nolint) shouldPassIssue(i *result.Issue) (bool, error) {
145108
return false, err
146109
}
147110

148-
if fd.isGenerated { // don't report issues for autogenerated files
149-
return false, nil
150-
}
151-
152111
for _, ir := range fd.ignoredRanges {
153112
if ir.doesMatch(i) {
154113
return false, nil

0 commit comments

Comments
 (0)