Skip to content

Commit 9eb569a

Browse files
committed
reduce mem. usage of unused and update staticcheck
The primary improvement is in early clearing of analyzed package's TypeInfo, facts, etc for whole program analyzers (`unused`). Clear it when it becomes unused and GC collects them early. Initially this clearing was performed for all analyzers except `unused`. Update staticcheck from v0.0.1-2019.2.3 to v0.0.1-2020.1.4 Also in this commit: * speed up loading packages from export data (2.5s -> 2.1s for std) by not using mutex for export data since it was allowed in x/tools#07722704da13 * make an order of execution of linters stable * update renameio and robustio * use robustio in caching Relates: #987, #994, #995, #1011
1 parent 02a4077 commit 9eb569a

File tree

16 files changed

+183
-147
lines changed

16 files changed

+183
-147
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,4 @@ require (
6060
// See https://github.com/golangci/golangci-lint/issues/995
6161
// Update only after mitigating this issue.
6262
// TODO: Enable back tests with skip "Issue955" after update.
63-
require honnef.co/go/tools v0.0.1-2019.2.3
63+
require honnef.co/go/tools v0.0.1-2020.1.3

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,11 +372,13 @@ golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtn
372372
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
373373
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
374374
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
375+
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
375376
golang.org/x/tools v0.0.0-20200228224639-71482053b885/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
376377
golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
377378
golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
378379
golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e h1:3Dzrrxi54Io7Aoyb0PYLsI47K2TxkRQg+cqUn+m04do=
379380
golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
381+
golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 h1:M9Fif0OxNji8w+HvmhVQ8KJtiZOsjU9RgslJGhn95XE=
380382
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
381383
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
382384
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
@@ -413,6 +415,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
413415
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
414416
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
415417
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
418+
honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U=
419+
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
416420
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
417421
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
418422
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=

internal/cache/cache.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"encoding/hex"
1515
"fmt"
1616
"io"
17-
"io/ioutil"
1817
"os"
1918
"path/filepath"
2019
"strconv"
@@ -24,6 +23,7 @@ import (
2423
"github.com/pkg/errors"
2524

2625
"github.com/golangci/golangci-lint/internal/renameio"
26+
"github.com/golangci/golangci-lint/internal/robustio"
2727
)
2828

2929
// An ActionID is a cache action key, the hash of a complete description of a
@@ -232,7 +232,7 @@ func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) {
232232
return nil, entry, err
233233
}
234234

235-
data, err := ioutil.ReadFile(outputFile)
235+
data, err := robustio.ReadFile(outputFile)
236236
if err != nil {
237237
return nil, entry, err
238238
}

internal/renameio/renameio_test.go

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// Use of this source code is governed by a BSD-style
33
// license that can be found in the LICENSE file.
44

5-
//+build !plan9
5+
// +build !plan9
66

77
package renameio
88

@@ -22,8 +22,6 @@ import (
2222
"github.com/golangci/golangci-lint/internal/robustio"
2323
)
2424

25-
const windowsOS = "windows"
26-
2725
func TestConcurrentReadsAndWrites(t *testing.T) {
2826
dir, err := ioutil.TempDir("", "renameio")
2927
if err != nil {
@@ -117,7 +115,7 @@ func TestConcurrentReadsAndWrites(t *testing.T) {
117115
}
118116

119117
var minWriteSuccesses int64 = attempts
120-
if runtime.GOOS == windowsOS {
118+
if runtime.GOOS == "windows" {
121119
// Windows produces frequent "Access is denied" errors under heavy rename load.
122120
// As long as those are the only errors and *some* of the writes succeed, we're happy.
123121
minWriteSuccesses = attempts / 4
@@ -130,10 +128,18 @@ func TestConcurrentReadsAndWrites(t *testing.T) {
130128
}
131129

132130
var minReadSuccesses int64 = attempts
133-
if runtime.GOOS == windowsOS {
131+
132+
switch runtime.GOOS {
133+
case "windows":
134134
// Windows produces frequent "Access is denied" errors under heavy rename load.
135-
// As long as those are the only errors and *some* of the writes succeed, we're happy.
135+
// As long as those are the only errors and *some* of the reads succeed, we're happy.
136136
minReadSuccesses = attempts / 4
137+
138+
case "darwin":
139+
// The filesystem on macOS 10.14 occasionally fails with "no such file or
140+
// directory" errors. See https://golang.org/issue/33041. The flake rate is
141+
// fairly low, so ensure that at least 75% of attempts succeed.
142+
minReadSuccesses = attempts - (attempts / 4)
137143
}
138144

139145
if readSuccesses < minReadSuccesses {

internal/renameio/umask_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// Use of this source code is governed by a BSD-style
33
// license that can be found in the LICENSE file.
44

5-
//+build !nacl,!plan9,!windows,!js
5+
// +build !plan9,!windows,!js
66

77
package renameio
88

@@ -19,6 +19,7 @@ func TestWriteFileModeAppliesUmask(t *testing.T) {
1919
if err != nil {
2020
t.Fatalf("Failed to create temporary directory: %v", err)
2121
}
22+
defer os.RemoveAll(dir)
2223

2324
const mode = 0644
2425
const umask = 0007
@@ -29,7 +30,6 @@ func TestWriteFileModeAppliesUmask(t *testing.T) {
2930
if err != nil {
3031
t.Fatalf("Failed to write file: %v", err)
3132
}
32-
defer os.RemoveAll(dir)
3333

3434
fi, err := os.Stat(file)
3535
if err != nil {

internal/robustio/robustio_darwin.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package robustio
6+
7+
import (
8+
"os"
9+
"syscall"
10+
)
11+
12+
const errFileNotFound = syscall.ENOENT
13+
14+
// isEphemeralError returns true if err may be resolved by waiting.
15+
func isEphemeralError(err error) bool {
16+
switch werr := err.(type) {
17+
case *os.PathError:
18+
err = werr.Err
19+
case *os.LinkError:
20+
err = werr.Err
21+
case *os.SyscallError:
22+
err = werr.Err
23+
24+
}
25+
if errno, ok := err.(syscall.Errno); ok {
26+
return errno == errFileNotFound
27+
}
28+
return false
29+
}

internal/robustio/robustio_flaky.go

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// +build windows darwin
6+
7+
package robustio
8+
9+
import (
10+
"io/ioutil"
11+
"math/rand"
12+
"os"
13+
"syscall"
14+
"time"
15+
)
16+
17+
const arbitraryTimeout = 500 * time.Millisecond
18+
19+
const ERROR_SHARING_VIOLATION = 32
20+
21+
// retry retries ephemeral errors from f up to an arbitrary timeout
22+
// to work around filesystem flakiness on Windows and Darwin.
23+
func retry(f func() (err error, mayRetry bool)) error {
24+
var (
25+
bestErr error
26+
lowestErrno syscall.Errno
27+
start time.Time
28+
nextSleep time.Duration = 1 * time.Millisecond
29+
)
30+
for {
31+
err, mayRetry := f()
32+
if err == nil || !mayRetry {
33+
return err
34+
}
35+
36+
if errno, ok := err.(syscall.Errno); ok && (lowestErrno == 0 || errno < lowestErrno) {
37+
bestErr = err
38+
lowestErrno = errno
39+
} else if bestErr == nil {
40+
bestErr = err
41+
}
42+
43+
if start.IsZero() {
44+
start = time.Now()
45+
} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
46+
break
47+
}
48+
time.Sleep(nextSleep)
49+
nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
50+
}
51+
52+
return bestErr
53+
}
54+
55+
// rename is like os.Rename, but retries ephemeral errors.
56+
//
57+
// On windows it wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
58+
// MOVEFILE_REPLACE_EXISTING.
59+
//
60+
// Windows also provides a different system call, ReplaceFile,
61+
// that provides similar semantics, but perhaps preserves more metadata. (The
62+
// documentation on the differences between the two is very sparse.)
63+
//
64+
// Empirical error rates with MoveFileEx are lower under modest concurrency, so
65+
// for now we're sticking with what the os package already provides.
66+
func rename(oldpath, newpath string) (err error) {
67+
return retry(func() (err error, mayRetry bool) {
68+
err = os.Rename(oldpath, newpath)
69+
return err, isEphemeralError(err)
70+
})
71+
}
72+
73+
// readFile is like ioutil.ReadFile, but retries ephemeral errors.
74+
func readFile(filename string) ([]byte, error) {
75+
var b []byte
76+
err := retry(func() (err error, mayRetry bool) {
77+
b, err = ioutil.ReadFile(filename)
78+
79+
// Unlike in rename, we do not retry errFileNotFound here: it can occur
80+
// as a spurious error, but the file may also genuinely not exist, so the
81+
// increase in robustness is probably not worth the extra latency.
82+
83+
return err, isEphemeralError(err) && err != errFileNotFound
84+
})
85+
return b, err
86+
}
87+
88+
func removeAll(path string) error {
89+
return retry(func() (err error, mayRetry bool) {
90+
err = os.RemoveAll(path)
91+
return err, isEphemeralError(err)
92+
})
93+
}

internal/robustio/robustio_other.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// Use of this source code is governed by a BSD-style
33
// license that can be found in the LICENSE file.
44

5-
//+build !windows
5+
//+build !windows,!darwin
66

77
package robustio
88

internal/robustio/robustio_windows.go

Lines changed: 9 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -5,93 +5,22 @@
55
package robustio
66

77
import (
8-
"io/ioutil"
9-
"math/rand"
108
"os"
119
"syscall"
12-
"time"
1310
)
1411

15-
const arbitraryTimeout = 500 * time.Millisecond
16-
17-
const ERROR_SHARING_VIOLATION = 32
18-
19-
// retry retries ephemeral errors from f up to an arbitrary timeout
20-
// to work around spurious filesystem errors on Windows
21-
func retry(f func() (err error, mayRetry bool)) error {
22-
var (
23-
bestErr error
24-
lowestErrno syscall.Errno
25-
start time.Time
26-
nextSleep time.Duration = 1 * time.Millisecond
27-
)
28-
for {
29-
err, mayRetry := f()
30-
if err == nil || !mayRetry {
31-
return err
32-
}
33-
34-
if errno, ok := err.(syscall.Errno); ok && (lowestErrno == 0 || errno < lowestErrno) {
35-
bestErr = err
36-
lowestErrno = errno
37-
} else if bestErr == nil {
38-
bestErr = err
39-
}
40-
41-
if start.IsZero() {
42-
start = time.Now()
43-
} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
44-
break
45-
}
46-
time.Sleep(nextSleep)
47-
nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
48-
}
49-
50-
return bestErr
51-
}
52-
53-
// rename is like os.Rename, but retries ephemeral errors.
54-
//
55-
// It wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
56-
// MOVEFILE_REPLACE_EXISTING.
57-
//
58-
// Windows also provides a different system call, ReplaceFile,
59-
// that provides similar semantics, but perhaps preserves more metadata. (The
60-
// documentation on the differences between the two is very sparse.)
61-
//
62-
// Empirical error rates with MoveFileEx are lower under modest concurrency, so
63-
// for now we're sticking with what the os package already provides.
64-
func rename(oldpath, newpath string) (err error) {
65-
return retry(func() (err error, mayRetry bool) {
66-
err = os.Rename(oldpath, newpath)
67-
return err, isEphemeralError(err)
68-
})
69-
}
70-
71-
// readFile is like ioutil.ReadFile, but retries ephemeral errors.
72-
func readFile(filename string) ([]byte, error) {
73-
var b []byte
74-
err := retry(func() (err error, mayRetry bool) {
75-
b, err = ioutil.ReadFile(filename)
76-
77-
// Unlike in rename, we do not retry ERROR_FILE_NOT_FOUND here: it can occur
78-
// as a spurious error, but the file may also genuinely not exist, so the
79-
// increase in robustness is probably not worth the extra latency.
80-
81-
return err, isEphemeralError(err) && err != syscall.ERROR_FILE_NOT_FOUND
82-
})
83-
return b, err
84-
}
85-
86-
func removeAll(path string) error {
87-
return retry(func() (err error, mayRetry bool) {
88-
err = os.RemoveAll(path)
89-
return err, isEphemeralError(err)
90-
})
91-
}
12+
const errFileNotFound = syscall.ERROR_FILE_NOT_FOUND
9213

9314
// isEphemeralError returns true if err may be resolved by waiting.
9415
func isEphemeralError(err error) bool {
16+
switch werr := err.(type) {
17+
case *os.PathError:
18+
err = werr.Err
19+
case *os.LinkError:
20+
err = werr.Err
21+
case *os.SyscallError:
22+
err = werr.Err
23+
}
9524
if errno, ok := err.(syscall.Errno); ok {
9625
switch errno {
9726
case syscall.ERROR_ACCESS_DENIED,

pkg/golinters/goanalysis/load/guard.go

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@ import (
77
)
88

99
type Guard struct {
10-
loadMutexes map[*packages.Package]*sync.Mutex
11-
mutexForExportData sync.Mutex
12-
mutex sync.Mutex
10+
loadMutexes map[*packages.Package]*sync.Mutex
11+
mutex sync.Mutex
1312
}
1413

1514
func NewGuard() *Guard {
@@ -26,10 +25,6 @@ func (g *Guard) MutexForPkg(pkg *packages.Package) *sync.Mutex {
2625
return g.loadMutexes[pkg]
2726
}
2827

29-
func (g *Guard) MutexForExportData() *sync.Mutex {
30-
return &g.mutexForExportData
31-
}
32-
3328
func (g *Guard) Mutex() *sync.Mutex {
3429
return &g.mutex
3530
}

0 commit comments

Comments
 (0)