diff --git a/.golangci.yml b/.golangci.yml index f9a821f6b31e..1d083481f909 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -221,11 +221,7 @@ issues: exclude-dirs: - test/testdata_etc # test files - - internal/cache # extracted from Go code - - internal/robustio # extracted from Go code - - internal/mmap # extracted from Go code - - internal/quoted # extracted from Go code - - internal/testenv # extracted from Go code + - internal/go # extracted from Go code run: timeout: 5m diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a59813236a3e..ed52fcf4ac43 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -1,663 +1,237 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package cache implements a build artifact cache. -// -// This package is a slightly modified fork of Go's -// cmd/go/internal/cache package. package cache import ( "bytes" - "crypto/sha256" + "encoding/gob" "encoding/hex" "errors" "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/golangci/golangci-lint/internal/mmap" - "github.com/golangci/golangci-lint/internal/robustio" - "github.com/rogpeppe/go-internal/lockedfile" -) - -// An ActionID is a cache action key, the hash of a complete description of a -// repeatable computation (command line, environment variables, -// input file contents, executable contents). -type ActionID [HashSize]byte - -// An OutputID is a cache output key, the hash of an output of a computation. -type OutputID [HashSize]byte - -// Cache is the interface as used by the cmd/go. -type Cache interface { - // Get returns the cache entry for the provided ActionID. - // On miss, the error type should be of type *entryNotFoundError. - // - // After a success call to Get, OutputFile(Entry.OutputID) must - // exist on disk for until Close is called (at the end of the process). - Get(ActionID) (Entry, error) - - // Put adds an item to the cache. - // - // The seeker is only used to seek to the beginning. After a call to Put, - // the seek position is not guaranteed to be in any particular state. - // - // As a special case, if the ReadSeeker is of type noVerifyReadSeeker, - // the verification from GODEBUG=goverifycache=1 is skipped. - // - // After a success call to Get, OutputFile(Entry.OutputID) must - // exist on disk for until Close is called (at the end of the process). - Put(ActionID, io.ReadSeeker) (_ OutputID, size int64, _ error) - - // Close is called at the end of the go process. Implementations can do - // cache cleanup work at this phase, or wait for and report any errors from - // background cleanup work started earlier. Any cache trimming should in one - // process should not violate cause the invariants of this interface to be - // violated in another process. Namely, a cache trim from one process should - // not delete an ObjectID from disk that was recently Get or Put from - // another process. As a rule of thumb, don't trim things used in the last - // day. - Close() error - - // OutputFile returns the path on disk where OutputID is stored. - // - // It's only called after a successful get or put call so it doesn't need - // to return an error; it's assumed that if the previous get or put succeeded, - // it's already on disk. - OutputFile(OutputID) string - - // FuzzDir returns where fuzz files are stored. - FuzzDir() string -} + "runtime" + "sort" + "sync" -// A Cache is a package cache, backed by a file system directory tree. -type DiskCache struct { - dir string - now func() time.Time -} + "golang.org/x/tools/go/packages" -// Open opens and returns the cache in the given directory. -// -// It is safe for multiple processes on a single machine to use the -// same cache directory in a local file system simultaneously. -// They will coordinate using operating system file locks and may -// duplicate effort but will not corrupt the cache. -// -// However, it is NOT safe for multiple processes on different machines -// to share a cache directory (for example, if the directory were stored -// in a network file system). File locking is notoriously unreliable in -// network file systems and may not suffice to protect the cache. -func Open(dir string) (*DiskCache, error) { - info, err := os.Stat(dir) - if err != nil { - return nil, err - } - if !info.IsDir() { - return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} - } - for i := 0; i < 256; i++ { - name := filepath.Join(dir, fmt.Sprintf("%02x", i)) - if err := os.MkdirAll(name, 0744); err != nil { - return nil, err - } - } - c := &DiskCache{ - dir: dir, - now: time.Now, - } - return c, nil -} - -// fileName returns the name of the file corresponding to the given id. -func (c *DiskCache) fileName(id [HashSize]byte, key string) string { - return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key) -} - -// An entryNotFoundError indicates that a cache entry was not found, with an -// optional underlying reason. -type entryNotFoundError struct { - Err error -} + "github.com/golangci/golangci-lint/internal/go/cache" + "github.com/golangci/golangci-lint/pkg/logutils" + "github.com/golangci/golangci-lint/pkg/timeutils" +) -func (e *entryNotFoundError) Error() string { - if e.Err == nil { - return "cache entry not found" - } - return fmt.Sprintf("cache entry not found: %v", e.Err) -} - -func (e *entryNotFoundError) Unwrap() error { - return e.Err -} +type HashMode int const ( - // action entry file is "v1 \n" - hexSize = HashSize * 2 - entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1 + HashModeNeedOnlySelf HashMode = iota + HashModeNeedDirectDeps + HashModeNeedAllDeps ) -// verify controls whether to run the cache in verify mode. -// In verify mode, the cache always returns errMissing from Get -// but then double-checks in Put that the data being written -// exactly matches any existing entry. This provides an easy -// way to detect program behavior that would have been different -// had the cache entry been returned from Get. -// -// verify is enabled by setting the environment variable -// GODEBUG=gocacheverify=1. -var verify = false - -var errVerifyMode = errors.New("gocacheverify=1") - -// DebugTest is set when GODEBUG=gocachetest=1 is in the environment. -var DebugTest = false - -// func init() { initEnv() } - -// var ( -// gocacheverify = godebug.New("gocacheverify") -// gocachehash = godebug.New("gocachehash") -// gocachetest = godebug.New("gocachetest") -// ) - -// func initEnv() { -// if gocacheverify.Value() == "1" { -// gocacheverify.IncNonDefault() -// verify = true -// } -// if gocachehash.Value() == "1" { -// gocachehash.IncNonDefault() -// debugHash = true -// } -// if gocachetest.Value() == "1" { -// gocachetest.IncNonDefault() -// DebugTest = true -// } -// } - -// Get looks up the action ID in the cache, -// returning the corresponding output ID and file size, if any. -// Note that finding an output ID does not guarantee that the -// saved file for that output ID is still available. -func (c *DiskCache) Get(id ActionID) (Entry, error) { - if verify { - return Entry{}, &entryNotFoundError{Err: errVerifyMode} - } - return c.get(id) -} - -type Entry struct { - OutputID OutputID - Size int64 - Time time.Time // when added to cache -} - -// get is Get but does not respect verify mode, so that Put can use it. -func (c *DiskCache) get(id ActionID) (Entry, error) { - missing := func(reason error) (Entry, error) { - return Entry{}, &entryNotFoundError{Err: reason} - } - f, err := os.Open(c.fileName(id, "a")) - if err != nil { - return missing(err) - } - defer f.Close() - entry := make([]byte, entrySize+1) // +1 to detect whether f is too long - if n, err := io.ReadFull(f, entry); n > entrySize { - return missing(errors.New("too long")) - } else if err != io.ErrUnexpectedEOF { - if err == io.EOF { - return missing(errors.New("file is empty")) +// Cache is a per-package data cache. A cached data is invalidated when +// package, or it's dependencies change. +type Cache struct { + lowLevelCache cache.Cache + pkgHashes sync.Map + sw *timeutils.Stopwatch + log logutils.Log + ioSem chan struct{} // semaphore limiting parallel IO +} + +func NewCache(sw *timeutils.Stopwatch, log logutils.Log) (*Cache, error) { + return &Cache{ + lowLevelCache: cache.Default(), + sw: sw, + log: log, + ioSem: make(chan struct{}, runtime.GOMAXPROCS(-1)), + }, nil +} + +func (c *Cache) Close() { + c.sw.TrackStage("close", func() { + err := c.lowLevelCache.Close() + if err != nil { + c.log.Errorf("cache close: %v", err) } - return missing(err) - } else if n < entrySize { - return missing(errors.New("entry file incomplete")) - } - if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' { - return missing(errors.New("invalid header")) - } - eid, entry := entry[3:3+hexSize], entry[3+hexSize:] - eout, entry := entry[1:1+hexSize], entry[1+hexSize:] - esize, entry := entry[1:1+20], entry[1+20:] - etime, entry := entry[1:1+20], entry[1+20:] - var buf [HashSize]byte - if _, err := hex.Decode(buf[:], eid); err != nil { - return missing(fmt.Errorf("decoding ID: %v", err)) - } else if buf != id { - return missing(errors.New("mismatched ID")) - } - if _, err := hex.Decode(buf[:], eout); err != nil { - return missing(fmt.Errorf("decoding output ID: %v", err)) - } - i := 0 - for i < len(esize) && esize[i] == ' ' { - i++ - } - size, err := strconv.ParseInt(string(esize[i:]), 10, 64) - if err != nil { - return missing(fmt.Errorf("parsing size: %v", err)) - } else if size < 0 { - return missing(errors.New("negative size")) - } - i = 0 - for i < len(etime) && etime[i] == ' ' { - i++ - } - tm, err := strconv.ParseInt(string(etime[i:]), 10, 64) - if err != nil { - return missing(fmt.Errorf("parsing timestamp: %v", err)) - } else if tm < 0 { - return missing(errors.New("negative timestamp")) - } - - err = c.used(c.fileName(id, "a")) - if err != nil { - return Entry{}, fmt.Errorf("failed to mark %s as used: %w", c.fileName(id, "a"), err) - } - - return Entry{buf, size, time.Unix(0, tm)}, nil + }) } -// GetFile looks up the action ID in the cache and returns -// the name of the corresponding data file. -func GetFile(c Cache, id ActionID) (file string, entry Entry, err error) { - entry, err = c.Get(id) - if err != nil { - return "", Entry{}, err - } - file = c.OutputFile(entry.OutputID) - info, err := os.Stat(file) +func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data any) error { + var err error + buf := &bytes.Buffer{} + c.sw.TrackStage("gob", func() { + err = gob.NewEncoder(buf).Encode(data) + }) if err != nil { - return "", Entry{}, &entryNotFoundError{Err: err} + return fmt.Errorf("failed to gob encode: %w", err) } - if info.Size() != entry.Size { - return "", Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} - } - return file, entry, nil -} -// GetBytes looks up the action ID in the cache and returns -// the corresponding output bytes. -// GetBytes should only be used for data that can be expected to fit in memory. -func GetBytes(c Cache, id ActionID) ([]byte, Entry, error) { - entry, err := c.Get(id) - if err != nil { - return nil, entry, err - } - data, err := robustio.ReadFile(c.OutputFile(entry.OutputID)) - if err != nil { - return nil, entry, err - } - if sha256.Sum256(data) != entry.OutputID { - return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")} - } - return data, entry, nil -} + var aID cache.ActionID -// GetMmap looks up the action ID in the cache and returns -// the corresponding output bytes. -// GetMmap should only be used for data that can be expected to fit in memory. -func GetMmap(c Cache, id ActionID) ([]byte, Entry, error) { - entry, err := c.Get(id) + c.sw.TrackStage("key build", func() { + aID, err = c.pkgActionID(pkg, mode) + if err == nil { + subkey, subkeyErr := cache.Subkey(aID, key) + if subkeyErr != nil { + err = fmt.Errorf("failed to build subkey: %w", subkeyErr) + } + aID = subkey + } + }) if err != nil { - return nil, entry, err + return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) } - md, err := mmap.Mmap(c.OutputFile(entry.OutputID)) + c.ioSem <- struct{}{} + c.sw.TrackStage("cache io", func() { + err = cache.PutBytes(c.lowLevelCache, aID, buf.Bytes()) + }) + <-c.ioSem if err != nil { - return nil, Entry{}, err - } - if int64(len(md.Data)) != entry.Size { - return nil, Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + return fmt.Errorf("failed to save data to low-level cache by key %s for package %s: %w", key, pkg.Name, err) } - return md.Data, entry, nil -} -// OutputFile returns the name of the cache file storing output with the given OutputID. -func (c *DiskCache) OutputFile(out OutputID) string { - file := c.fileName(out, "d") - c.used(file) - return file + return nil } -// Time constants for cache expiration. -// -// We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour), -// to avoid causing many unnecessary inode updates. The mtimes therefore -// roughly reflect "time of last use" but may in fact be older by at most an hour. -// -// We scan the cache for entries to delete at most once per trimInterval (1 day). -// -// When we do scan the cache, we delete entries that have not been used for -// at least trimLimit (5 days). Statistics gathered from a month of usage by -// Go developers found that essentially all reuse of cached entries happened -// within 5 days of the previous reuse. See golang.org/issue/22990. -const ( - mtimeInterval = 1 * time.Hour - trimInterval = 24 * time.Hour - trimLimit = 5 * 24 * time.Hour -) - -// used makes a best-effort attempt to update mtime on file, -// so that mtime reflects cache access time. -// -// Because the reflection only needs to be approximate, -// and to reduce the amount of disk activity caused by using -// cache entries, used only updates the mtime if the current -// mtime is more than an hour old. This heuristic eliminates -// nearly all of the mtime updates that would otherwise happen, -// while still keeping the mtimes useful for cache trimming. -func (c *DiskCache) used(file string) error { - info, err := os.Stat(file) - if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval { - return nil - } +var ErrMissing = errors.New("missing data") - if err != nil { - if os.IsNotExist(err) { - return &entryNotFoundError{Err: err} +func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data any) error { + var aID cache.ActionID + var err error + c.sw.TrackStage("key build", func() { + aID, err = c.pkgActionID(pkg, mode) + if err == nil { + subkey, subkeyErr := cache.Subkey(aID, key) + if subkeyErr != nil { + err = fmt.Errorf("failed to build subkey: %w", subkeyErr) + } + aID = subkey } - return &entryNotFoundError{Err: fmt.Errorf("failed to stat file %s: %w", file, err)} - } - - err = os.Chtimes(file, c.now(), c.now()) + }) if err != nil { - return fmt.Errorf("failed to change time of file %s: %w", file, err) + return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) } - return nil -} - -func (c *DiskCache) Close() error { return c.Trim() } - -// Trim removes old cache entries that are likely not to be reused. -func (c *DiskCache) Trim() error { - now := c.now() - - // We maintain in dir/trim.txt the time of the last completed cache trim. - // If the cache has been trimmed recently enough, do nothing. - // This is the common case. - // If the trim file is corrupt, detected if the file can't be parsed, or the - // trim time is too far in the future, attempt the trim anyway. It's possible that - // the cache was full when the corruption happened. Attempting a trim on - // an empty cache is cheap, so there wouldn't be a big performance hit in that case. - if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil { - if t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil { - lastTrim := time.Unix(t, 0) - if d := now.Sub(lastTrim); d < trimInterval && d > -mtimeInterval { - return nil - } + var b []byte + c.ioSem <- struct{}{} + c.sw.TrackStage("cache io", func() { + b, _, err = cache.GetBytes(c.lowLevelCache, aID) + }) + <-c.ioSem + if err != nil { + if cache.IsErrMissing(err) { + return ErrMissing } + return fmt.Errorf("failed to get data from low-level cache by key %s for package %s: %w", key, pkg.Name, err) } - // Trim each of the 256 subdirectories. - // We subtract an additional mtimeInterval - // to account for the imprecision of our "last used" mtimes. - cutoff := now.Add(-trimLimit - mtimeInterval) - for i := 0; i < 256; i++ { - subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i)) - c.trimSubdir(subdir, cutoff) - } - - // Ignore errors from here: if we don't write the complete timestamp, the - // cache will appear older than it is, and we'll trim it again next time. - var b bytes.Buffer - fmt.Fprintf(&b, "%d", now.Unix()) - if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0666); err != nil { - return err + c.sw.TrackStage("gob", func() { + err = gob.NewDecoder(bytes.NewReader(b)).Decode(data) + }) + if err != nil { + return fmt.Errorf("failed to gob decode: %w", err) } return nil } -// trimSubdir trims a single cache subdirectory. -func (c *DiskCache) trimSubdir(subdir string, cutoff time.Time) { - // Read all directory entries from subdir before removing - // any files, in case removing files invalidates the file offset - // in the directory scan. Also, ignore error from f.Readdirnames, - // because we don't care about reporting the error and we still - // want to process any entries found before the error. - f, err := os.Open(subdir) +func (c *Cache) pkgActionID(pkg *packages.Package, mode HashMode) (cache.ActionID, error) { + hash, err := c.packageHash(pkg, mode) if err != nil { - return - } - names, _ := f.Readdirnames(-1) - f.Close() - - for _, name := range names { - // Remove only cache entries (xxxx-a and xxxx-d). - if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") { - continue - } - entry := filepath.Join(subdir, name) - info, err := os.Stat(entry) - if err == nil && info.ModTime().Before(cutoff) { - os.Remove(entry) - } - } -} - -// putIndexEntry adds an entry to the cache recording that executing the action -// with the given id produces an output with the given output id (hash) and size. -func (c *DiskCache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error { - // Note: We expect that for one reason or another it may happen - // that repeating an action produces a different output hash - // (for example, if the output contains a time stamp or temp dir name). - // While not ideal, this is also not a correctness problem, so we - // don't make a big deal about it. In particular, we leave the action - // cache entries writable specifically so that they can be overwritten. - // - // Setting GODEBUG=gocacheverify=1 does make a big deal: - // in verify mode we are double-checking that the cache entries - // are entirely reproducible. As just noted, this may be unrealistic - // in some cases but the check is also useful for shaking out real bugs. - entry := fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()) - if verify && allowVerify { - old, err := c.get(id) - if err == nil && (old.OutputID != out || old.Size != size) { - // panic to show stack trace, so we can see what code is generating this cache entry. - msg := fmt.Sprintf("go: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size) - panic(msg) - } + return cache.ActionID{}, fmt.Errorf("failed to get package hash: %w", err) } - file := c.fileName(id, "a") - // Copy file to cache directory. - mode := os.O_WRONLY | os.O_CREATE - f, err := os.OpenFile(file, mode, 0666) + key, err := cache.NewHash("action ID") if err != nil { - return err - } - _, err = f.WriteString(entry) - if err == nil { - // Truncate the file only *after* writing it. - // (This should be a no-op, but truncate just in case of previous corruption.) - // - // This differs from os.WriteFile, which truncates to 0 *before* writing - // via os.O_TRUNC. Truncating only after writing ensures that a second write - // of the same content to the same file is idempotent, and does not — even - // temporarily! — undo the effect of the first write. - err = f.Truncate(int64(len(entry))) - } - if closeErr := f.Close(); err == nil { - err = closeErr + return cache.ActionID{}, fmt.Errorf("failed to make a hash: %w", err) } - if err != nil { - // TODO(bcmills): This Remove potentially races with another go command writing to file. - // Can we eliminate it? - os.Remove(file) - return err - } - err = os.Chtimes(file, c.now(), c.now()) // mainly for tests - if err != nil { - return fmt.Errorf("failed to change time of file %s: %w", file, err) - } - - return nil -} + fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) + fmt.Fprintf(key, "pkghash %s\n", hash) -// noVerifyReadSeeker is an io.ReadSeeker wrapper sentinel type -// that says that Cache.Put should skip the verify check -// (from GODEBUG=goverifycache=1). -type noVerifyReadSeeker struct { - io.ReadSeeker + return key.Sum(), nil } -// Put stores the given output in the cache as the output for the action ID. -// It may read file twice. The content of file must not change between the two passes. -func (c *DiskCache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { - wrapper, isNoVerify := file.(noVerifyReadSeeker) - if isNoVerify { - file = wrapper.ReadSeeker +// packageHash computes a package's hash. The hash is based on all Go +// files that make up the package, as well as the hashes of imported +// packages. +func (c *Cache) packageHash(pkg *packages.Package, mode HashMode) (string, error) { + type hashResults map[HashMode]string + hashResI, ok := c.pkgHashes.Load(pkg) + if ok { + hashRes := hashResI.(hashResults) + if _, ok := hashRes[mode]; !ok { + return "", fmt.Errorf("no mode %d in hash result", mode) + } + return hashRes[mode], nil } - return c.put(id, file, !isNoVerify) -} -// PutNoVerify is like Put but disables the verify check -// when GODEBUG=goverifycache=1 is set. -// It is meant for data that is OK to cache but that we expect to vary slightly from run to run, -// like test output containing times and the like. -func PutNoVerify(c Cache, id ActionID, file io.ReadSeeker) (OutputID, int64, error) { - return c.Put(id, noVerifyReadSeeker{file}) -} + hashRes := hashResults{} -func (c *DiskCache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { - // Compute output ID. - h := sha256.New() - if _, err := file.Seek(0, 0); err != nil { - return OutputID{}, 0, err - } - size, err := io.Copy(h, file) + key, err := cache.NewHash("package hash") if err != nil { - return OutputID{}, 0, err + return "", fmt.Errorf("failed to make a hash: %w", err) } - var out OutputID - h.Sum(out[:0]) - // Copy to cached output file (if not already present). - if err := c.copyFile(file, out, size); err != nil { - return out, size, err + fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) + for _, f := range pkg.CompiledGoFiles { + c.ioSem <- struct{}{} + h, fErr := cache.FileHash(f) + <-c.ioSem + if fErr != nil { + return "", fmt.Errorf("failed to calculate file %s hash: %w", f, fErr) + } + fmt.Fprintf(key, "file %s %x\n", f, h) } + curSum := key.Sum() + hashRes[HashModeNeedOnlySelf] = hex.EncodeToString(curSum[:]) - // Add to cache index. - return out, size, c.putIndexEntry(id, out, size, allowVerify) -} - -// PutBytes stores the given bytes in the cache as the output for the action ID. -func PutBytes(c Cache, id ActionID, data []byte) error { - _, _, err := c.Put(id, bytes.NewReader(data)) - return err -} + imps := make([]*packages.Package, 0, len(pkg.Imports)) + for _, imp := range pkg.Imports { + imps = append(imps, imp) + } + sort.Slice(imps, func(i, j int) bool { + return imps[i].PkgPath < imps[j].PkgPath + }) -// copyFile copies file into the cache, expecting it to have the given -// output ID and size, if that file is not present already. -func (c *DiskCache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { - name := c.fileName(out, "d") - info, err := os.Stat(name) - if err == nil && info.Size() == size { - // Check hash. - if f, err := os.Open(name); err == nil { - h := sha256.New() - _, copyErr := io.Copy(h, f) - if copyErr != nil { - return fmt.Errorf("failed to copy to sha256: %w", copyErr) + calcDepsHash := func(depMode HashMode) error { + for _, dep := range imps { + if dep.PkgPath == "unsafe" { + continue } - f.Close() - var out2 OutputID - h.Sum(out2[:0]) - if out == out2 { - return nil + depHash, depErr := c.packageHash(dep, depMode) + if depErr != nil { + return fmt.Errorf("failed to calculate hash for dependency %s with mode %d: %w", dep.Name, depMode, depErr) } + + fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, depHash) } - // Hash did not match. Fall through and rewrite file. + return nil } - // Copy file to cache directory. - mode := os.O_RDWR | os.O_CREATE - if err == nil && info.Size() > size { // shouldn't happen but fix in case - mode |= os.O_TRUNC - } - f, err := os.OpenFile(name, mode, 0666) - if err != nil { - return err - } - defer f.Close() - if size == 0 { - // File now exists with correct size. - // Only one possible zero-length file, so contents are OK too. - // Early return here makes sure there's a "last byte" for code below. - return nil + if err := calcDepsHash(HashModeNeedOnlySelf); err != nil { + return "", err } - // From here on, if any of the I/O writing the file fails, - // we make a best-effort attempt to truncate the file f - // before returning, to avoid leaving bad bytes in the file. + curSum = key.Sum() + hashRes[HashModeNeedDirectDeps] = hex.EncodeToString(curSum[:]) - // Copy file to f, but also into h to double-check hash. - if _, err := file.Seek(0, 0); err != nil { - f.Truncate(0) - return err - } - h := sha256.New() - w := io.MultiWriter(f, h) - if _, err := io.CopyN(w, file, size-1); err != nil { - f.Truncate(0) - return err - } - // Check last byte before writing it; writing it will make the size match - // what other processes expect to find and might cause them to start - // using the file. - buf := make([]byte, 1) - if _, err := file.Read(buf); err != nil { - f.Truncate(0) - return err - } - n, wErr := h.Write(buf) - if n != len(buf) { - return fmt.Errorf("wrote to hash %d/%d bytes with error %w", n, len(buf), wErr) + if err := calcDepsHash(HashModeNeedAllDeps); err != nil { + return "", err } + curSum = key.Sum() + hashRes[HashModeNeedAllDeps] = hex.EncodeToString(curSum[:]) - sum := h.Sum(nil) - if !bytes.Equal(sum, out[:]) { - f.Truncate(0) - return fmt.Errorf("file content changed underfoot") + if _, ok := hashRes[mode]; !ok { + return "", fmt.Errorf("invalid mode %d", mode) } - // Commit cache file entry. - if _, err := f.Write(buf); err != nil { - f.Truncate(0) - return err - } - if err := f.Close(); err != nil { - // Data might not have been written, - // but file may look like it is the right size. - // To be extra careful, remove cached file. - os.Remove(name) - return err - } - err = os.Chtimes(name, c.now(), c.now()) // mainly for tests - if err != nil { - return fmt.Errorf("failed to change time of file %s: %w", name, err) - } + c.pkgHashes.Store(pkg, hashRes) + return hashRes[mode], nil +} - return nil +func SetSalt(b *bytes.Buffer) { + cache.SetSalt(b.Bytes()) } -// FuzzDir returns a subdirectory within the cache for storing fuzzing data. -// The subdirectory may not exist. -// -// This directory is managed by the internal/fuzz package. Files in this -// directory aren't removed by the 'go clean -cache' command or by Trim. -// They may be removed with 'go clean -fuzzcache'. -// -// TODO(#48526): make Trim remove unused files from this directory. -func (c *DiskCache) FuzzDir() string { - return filepath.Join(c.dir, "fuzz") +func DefaultDir() string { + cacheDir, _ := cache.DefaultDir() + return cacheDir } diff --git a/internal/cache/LICENSE b/internal/go/LICENSE similarity index 100% rename from internal/cache/LICENSE rename to internal/go/LICENSE diff --git a/internal/go/cache/cache.go b/internal/go/cache/cache.go new file mode 100644 index 000000000000..7bf4f1d660e5 --- /dev/null +++ b/internal/go/cache/cache.go @@ -0,0 +1,663 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cache implements a build artifact cache. +// +// This package is a slightly modified fork of Go's +// cmd/go/internal/cache package. +package cache + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/golangci/golangci-lint/internal/go/mmap" + "github.com/golangci/golangci-lint/internal/go/robustio" + "github.com/rogpeppe/go-internal/lockedfile" +) + +// An ActionID is a cache action key, the hash of a complete description of a +// repeatable computation (command line, environment variables, +// input file contents, executable contents). +type ActionID [HashSize]byte + +// An OutputID is a cache output key, the hash of an output of a computation. +type OutputID [HashSize]byte + +// Cache is the interface as used by the cmd/go. +type Cache interface { + // Get returns the cache entry for the provided ActionID. + // On miss, the error type should be of type *entryNotFoundError. + // + // After a success call to Get, OutputFile(Entry.OutputID) must + // exist on disk for until Close is called (at the end of the process). + Get(ActionID) (Entry, error) + + // Put adds an item to the cache. + // + // The seeker is only used to seek to the beginning. After a call to Put, + // the seek position is not guaranteed to be in any particular state. + // + // As a special case, if the ReadSeeker is of type noVerifyReadSeeker, + // the verification from GODEBUG=goverifycache=1 is skipped. + // + // After a success call to Get, OutputFile(Entry.OutputID) must + // exist on disk for until Close is called (at the end of the process). + Put(ActionID, io.ReadSeeker) (_ OutputID, size int64, _ error) + + // Close is called at the end of the go process. Implementations can do + // cache cleanup work at this phase, or wait for and report any errors from + // background cleanup work started earlier. Any cache trimming should in one + // process should not violate cause the invariants of this interface to be + // violated in another process. Namely, a cache trim from one process should + // not delete an ObjectID from disk that was recently Get or Put from + // another process. As a rule of thumb, don't trim things used in the last + // day. + Close() error + + // OutputFile returns the path on disk where OutputID is stored. + // + // It's only called after a successful get or put call so it doesn't need + // to return an error; it's assumed that if the previous get or put succeeded, + // it's already on disk. + OutputFile(OutputID) string + + // FuzzDir returns where fuzz files are stored. + FuzzDir() string +} + +// A Cache is a package cache, backed by a file system directory tree. +type DiskCache struct { + dir string + now func() time.Time +} + +// Open opens and returns the cache in the given directory. +// +// It is safe for multiple processes on a single machine to use the +// same cache directory in a local file system simultaneously. +// They will coordinate using operating system file locks and may +// duplicate effort but will not corrupt the cache. +// +// However, it is NOT safe for multiple processes on different machines +// to share a cache directory (for example, if the directory were stored +// in a network file system). File locking is notoriously unreliable in +// network file systems and may not suffice to protect the cache. +func Open(dir string) (*DiskCache, error) { + info, err := os.Stat(dir) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} + } + for i := 0; i < 256; i++ { + name := filepath.Join(dir, fmt.Sprintf("%02x", i)) + if err := os.MkdirAll(name, 0744); err != nil { + return nil, err + } + } + c := &DiskCache{ + dir: dir, + now: time.Now, + } + return c, nil +} + +// fileName returns the name of the file corresponding to the given id. +func (c *DiskCache) fileName(id [HashSize]byte, key string) string { + return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key) +} + +// An entryNotFoundError indicates that a cache entry was not found, with an +// optional underlying reason. +type entryNotFoundError struct { + Err error +} + +func (e *entryNotFoundError) Error() string { + if e.Err == nil { + return "cache entry not found" + } + return fmt.Sprintf("cache entry not found: %v", e.Err) +} + +func (e *entryNotFoundError) Unwrap() error { + return e.Err +} + +const ( + // action entry file is "v1 \n" + hexSize = HashSize * 2 + entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1 +) + +// verify controls whether to run the cache in verify mode. +// In verify mode, the cache always returns errMissing from Get +// but then double-checks in Put that the data being written +// exactly matches any existing entry. This provides an easy +// way to detect program behavior that would have been different +// had the cache entry been returned from Get. +// +// verify is enabled by setting the environment variable +// GODEBUG=gocacheverify=1. +var verify = false + +var errVerifyMode = errors.New("gocacheverify=1") + +// DebugTest is set when GODEBUG=gocachetest=1 is in the environment. +var DebugTest = false + +// func init() { initEnv() } + +// var ( +// gocacheverify = godebug.New("gocacheverify") +// gocachehash = godebug.New("gocachehash") +// gocachetest = godebug.New("gocachetest") +// ) + +// func initEnv() { +// if gocacheverify.Value() == "1" { +// gocacheverify.IncNonDefault() +// verify = true +// } +// if gocachehash.Value() == "1" { +// gocachehash.IncNonDefault() +// debugHash = true +// } +// if gocachetest.Value() == "1" { +// gocachetest.IncNonDefault() +// DebugTest = true +// } +// } + +// Get looks up the action ID in the cache, +// returning the corresponding output ID and file size, if any. +// Note that finding an output ID does not guarantee that the +// saved file for that output ID is still available. +func (c *DiskCache) Get(id ActionID) (Entry, error) { + if verify { + return Entry{}, &entryNotFoundError{Err: errVerifyMode} + } + return c.get(id) +} + +type Entry struct { + OutputID OutputID + Size int64 + Time time.Time // when added to cache +} + +// get is Get but does not respect verify mode, so that Put can use it. +func (c *DiskCache) get(id ActionID) (Entry, error) { + missing := func(reason error) (Entry, error) { + return Entry{}, &entryNotFoundError{Err: reason} + } + f, err := os.Open(c.fileName(id, "a")) + if err != nil { + return missing(err) + } + defer f.Close() + entry := make([]byte, entrySize+1) // +1 to detect whether f is too long + if n, err := io.ReadFull(f, entry); n > entrySize { + return missing(errors.New("too long")) + } else if err != io.ErrUnexpectedEOF { + if err == io.EOF { + return missing(errors.New("file is empty")) + } + return missing(err) + } else if n < entrySize { + return missing(errors.New("entry file incomplete")) + } + if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' { + return missing(errors.New("invalid header")) + } + eid, entry := entry[3:3+hexSize], entry[3+hexSize:] + eout, entry := entry[1:1+hexSize], entry[1+hexSize:] + esize, entry := entry[1:1+20], entry[1+20:] + etime, entry := entry[1:1+20], entry[1+20:] + var buf [HashSize]byte + if _, err := hex.Decode(buf[:], eid); err != nil { + return missing(fmt.Errorf("decoding ID: %v", err)) + } else if buf != id { + return missing(errors.New("mismatched ID")) + } + if _, err := hex.Decode(buf[:], eout); err != nil { + return missing(fmt.Errorf("decoding output ID: %v", err)) + } + i := 0 + for i < len(esize) && esize[i] == ' ' { + i++ + } + size, err := strconv.ParseInt(string(esize[i:]), 10, 64) + if err != nil { + return missing(fmt.Errorf("parsing size: %v", err)) + } else if size < 0 { + return missing(errors.New("negative size")) + } + i = 0 + for i < len(etime) && etime[i] == ' ' { + i++ + } + tm, err := strconv.ParseInt(string(etime[i:]), 10, 64) + if err != nil { + return missing(fmt.Errorf("parsing timestamp: %v", err)) + } else if tm < 0 { + return missing(errors.New("negative timestamp")) + } + + err = c.used(c.fileName(id, "a")) + if err != nil { + return Entry{}, fmt.Errorf("failed to mark %s as used: %w", c.fileName(id, "a"), err) + } + + return Entry{buf, size, time.Unix(0, tm)}, nil +} + +// GetFile looks up the action ID in the cache and returns +// the name of the corresponding data file. +func GetFile(c Cache, id ActionID) (file string, entry Entry, err error) { + entry, err = c.Get(id) + if err != nil { + return "", Entry{}, err + } + file = c.OutputFile(entry.OutputID) + info, err := os.Stat(file) + if err != nil { + return "", Entry{}, &entryNotFoundError{Err: err} + } + if info.Size() != entry.Size { + return "", Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + } + return file, entry, nil +} + +// GetBytes looks up the action ID in the cache and returns +// the corresponding output bytes. +// GetBytes should only be used for data that can be expected to fit in memory. +func GetBytes(c Cache, id ActionID) ([]byte, Entry, error) { + entry, err := c.Get(id) + if err != nil { + return nil, entry, err + } + data, err := robustio.ReadFile(c.OutputFile(entry.OutputID)) + if err != nil { + return nil, entry, err + } + if sha256.Sum256(data) != entry.OutputID { + return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")} + } + return data, entry, nil +} + +// GetMmap looks up the action ID in the cache and returns +// the corresponding output bytes. +// GetMmap should only be used for data that can be expected to fit in memory. +func GetMmap(c Cache, id ActionID) ([]byte, Entry, error) { + entry, err := c.Get(id) + if err != nil { + return nil, entry, err + } + md, err := mmap.Mmap(c.OutputFile(entry.OutputID)) + if err != nil { + return nil, Entry{}, err + } + if int64(len(md.Data)) != entry.Size { + return nil, Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + } + return md.Data, entry, nil +} + +// OutputFile returns the name of the cache file storing output with the given OutputID. +func (c *DiskCache) OutputFile(out OutputID) string { + file := c.fileName(out, "d") + c.used(file) + return file +} + +// Time constants for cache expiration. +// +// We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour), +// to avoid causing many unnecessary inode updates. The mtimes therefore +// roughly reflect "time of last use" but may in fact be older by at most an hour. +// +// We scan the cache for entries to delete at most once per trimInterval (1 day). +// +// When we do scan the cache, we delete entries that have not been used for +// at least trimLimit (5 days). Statistics gathered from a month of usage by +// Go developers found that essentially all reuse of cached entries happened +// within 5 days of the previous reuse. See golang.org/issue/22990. +const ( + mtimeInterval = 1 * time.Hour + trimInterval = 24 * time.Hour + trimLimit = 5 * 24 * time.Hour +) + +// used makes a best-effort attempt to update mtime on file, +// so that mtime reflects cache access time. +// +// Because the reflection only needs to be approximate, +// and to reduce the amount of disk activity caused by using +// cache entries, used only updates the mtime if the current +// mtime is more than an hour old. This heuristic eliminates +// nearly all of the mtime updates that would otherwise happen, +// while still keeping the mtimes useful for cache trimming. +func (c *DiskCache) used(file string) error { + info, err := os.Stat(file) + if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval { + return nil + } + + if err != nil { + if os.IsNotExist(err) { + return &entryNotFoundError{Err: err} + } + return &entryNotFoundError{Err: fmt.Errorf("failed to stat file %s: %w", file, err)} + } + + err = os.Chtimes(file, c.now(), c.now()) + if err != nil { + return fmt.Errorf("failed to change time of file %s: %w", file, err) + } + + return nil +} + +func (c *DiskCache) Close() error { return c.Trim() } + +// Trim removes old cache entries that are likely not to be reused. +func (c *DiskCache) Trim() error { + now := c.now() + + // We maintain in dir/trim.txt the time of the last completed cache trim. + // If the cache has been trimmed recently enough, do nothing. + // This is the common case. + // If the trim file is corrupt, detected if the file can't be parsed, or the + // trim time is too far in the future, attempt the trim anyway. It's possible that + // the cache was full when the corruption happened. Attempting a trim on + // an empty cache is cheap, so there wouldn't be a big performance hit in that case. + if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil { + if t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil { + lastTrim := time.Unix(t, 0) + if d := now.Sub(lastTrim); d < trimInterval && d > -mtimeInterval { + return nil + } + } + } + + // Trim each of the 256 subdirectories. + // We subtract an additional mtimeInterval + // to account for the imprecision of our "last used" mtimes. + cutoff := now.Add(-trimLimit - mtimeInterval) + for i := 0; i < 256; i++ { + subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i)) + c.trimSubdir(subdir, cutoff) + } + + // Ignore errors from here: if we don't write the complete timestamp, the + // cache will appear older than it is, and we'll trim it again next time. + var b bytes.Buffer + fmt.Fprintf(&b, "%d", now.Unix()) + if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0666); err != nil { + return err + } + + return nil +} + +// trimSubdir trims a single cache subdirectory. +func (c *DiskCache) trimSubdir(subdir string, cutoff time.Time) { + // Read all directory entries from subdir before removing + // any files, in case removing files invalidates the file offset + // in the directory scan. Also, ignore error from f.Readdirnames, + // because we don't care about reporting the error and we still + // want to process any entries found before the error. + f, err := os.Open(subdir) + if err != nil { + return + } + names, _ := f.Readdirnames(-1) + f.Close() + + for _, name := range names { + // Remove only cache entries (xxxx-a and xxxx-d). + if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") { + continue + } + entry := filepath.Join(subdir, name) + info, err := os.Stat(entry) + if err == nil && info.ModTime().Before(cutoff) { + os.Remove(entry) + } + } +} + +// putIndexEntry adds an entry to the cache recording that executing the action +// with the given id produces an output with the given output id (hash) and size. +func (c *DiskCache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error { + // Note: We expect that for one reason or another it may happen + // that repeating an action produces a different output hash + // (for example, if the output contains a time stamp or temp dir name). + // While not ideal, this is also not a correctness problem, so we + // don't make a big deal about it. In particular, we leave the action + // cache entries writable specifically so that they can be overwritten. + // + // Setting GODEBUG=gocacheverify=1 does make a big deal: + // in verify mode we are double-checking that the cache entries + // are entirely reproducible. As just noted, this may be unrealistic + // in some cases but the check is also useful for shaking out real bugs. + entry := fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()) + if verify && allowVerify { + old, err := c.get(id) + if err == nil && (old.OutputID != out || old.Size != size) { + // panic to show stack trace, so we can see what code is generating this cache entry. + msg := fmt.Sprintf("go: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size) + panic(msg) + } + } + file := c.fileName(id, "a") + + // Copy file to cache directory. + mode := os.O_WRONLY | os.O_CREATE + f, err := os.OpenFile(file, mode, 0666) + if err != nil { + return err + } + _, err = f.WriteString(entry) + if err == nil { + // Truncate the file only *after* writing it. + // (This should be a no-op, but truncate just in case of previous corruption.) + // + // This differs from os.WriteFile, which truncates to 0 *before* writing + // via os.O_TRUNC. Truncating only after writing ensures that a second write + // of the same content to the same file is idempotent, and does not — even + // temporarily! — undo the effect of the first write. + err = f.Truncate(int64(len(entry))) + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + // TODO(bcmills): This Remove potentially races with another go command writing to file. + // Can we eliminate it? + os.Remove(file) + return err + } + err = os.Chtimes(file, c.now(), c.now()) // mainly for tests + if err != nil { + return fmt.Errorf("failed to change time of file %s: %w", file, err) + } + + return nil +} + +// noVerifyReadSeeker is an io.ReadSeeker wrapper sentinel type +// that says that Cache.Put should skip the verify check +// (from GODEBUG=goverifycache=1). +type noVerifyReadSeeker struct { + io.ReadSeeker +} + +// Put stores the given output in the cache as the output for the action ID. +// It may read file twice. The content of file must not change between the two passes. +func (c *DiskCache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { + wrapper, isNoVerify := file.(noVerifyReadSeeker) + if isNoVerify { + file = wrapper.ReadSeeker + } + return c.put(id, file, !isNoVerify) +} + +// PutNoVerify is like Put but disables the verify check +// when GODEBUG=goverifycache=1 is set. +// It is meant for data that is OK to cache but that we expect to vary slightly from run to run, +// like test output containing times and the like. +func PutNoVerify(c Cache, id ActionID, file io.ReadSeeker) (OutputID, int64, error) { + return c.Put(id, noVerifyReadSeeker{file}) +} + +func (c *DiskCache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { + // Compute output ID. + h := sha256.New() + if _, err := file.Seek(0, 0); err != nil { + return OutputID{}, 0, err + } + size, err := io.Copy(h, file) + if err != nil { + return OutputID{}, 0, err + } + var out OutputID + h.Sum(out[:0]) + + // Copy to cached output file (if not already present). + if err := c.copyFile(file, out, size); err != nil { + return out, size, err + } + + // Add to cache index. + return out, size, c.putIndexEntry(id, out, size, allowVerify) +} + +// PutBytes stores the given bytes in the cache as the output for the action ID. +func PutBytes(c Cache, id ActionID, data []byte) error { + _, _, err := c.Put(id, bytes.NewReader(data)) + return err +} + +// copyFile copies file into the cache, expecting it to have the given +// output ID and size, if that file is not present already. +func (c *DiskCache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { + name := c.fileName(out, "d") + info, err := os.Stat(name) + if err == nil && info.Size() == size { + // Check hash. + if f, err := os.Open(name); err == nil { + h := sha256.New() + _, copyErr := io.Copy(h, f) + if copyErr != nil { + return fmt.Errorf("failed to copy to sha256: %w", copyErr) + } + + f.Close() + var out2 OutputID + h.Sum(out2[:0]) + if out == out2 { + return nil + } + } + // Hash did not match. Fall through and rewrite file. + } + + // Copy file to cache directory. + mode := os.O_RDWR | os.O_CREATE + if err == nil && info.Size() > size { // shouldn't happen but fix in case + mode |= os.O_TRUNC + } + f, err := os.OpenFile(name, mode, 0666) + if err != nil { + return err + } + defer f.Close() + if size == 0 { + // File now exists with correct size. + // Only one possible zero-length file, so contents are OK too. + // Early return here makes sure there's a "last byte" for code below. + return nil + } + + // From here on, if any of the I/O writing the file fails, + // we make a best-effort attempt to truncate the file f + // before returning, to avoid leaving bad bytes in the file. + + // Copy file to f, but also into h to double-check hash. + if _, err := file.Seek(0, 0); err != nil { + f.Truncate(0) + return err + } + h := sha256.New() + w := io.MultiWriter(f, h) + if _, err := io.CopyN(w, file, size-1); err != nil { + f.Truncate(0) + return err + } + // Check last byte before writing it; writing it will make the size match + // what other processes expect to find and might cause them to start + // using the file. + buf := make([]byte, 1) + if _, err := file.Read(buf); err != nil { + f.Truncate(0) + return err + } + n, wErr := h.Write(buf) + if n != len(buf) { + return fmt.Errorf("wrote to hash %d/%d bytes with error %w", n, len(buf), wErr) + } + + sum := h.Sum(nil) + if !bytes.Equal(sum, out[:]) { + f.Truncate(0) + return fmt.Errorf("file content changed underfoot") + } + + // Commit cache file entry. + if _, err := f.Write(buf); err != nil { + f.Truncate(0) + return err + } + if err := f.Close(); err != nil { + // Data might not have been written, + // but file may look like it is the right size. + // To be extra careful, remove cached file. + os.Remove(name) + return err + } + err = os.Chtimes(name, c.now(), c.now()) // mainly for tests + if err != nil { + return fmt.Errorf("failed to change time of file %s: %w", name, err) + } + + return nil +} + +// FuzzDir returns a subdirectory within the cache for storing fuzzing data. +// The subdirectory may not exist. +// +// This directory is managed by the internal/fuzz package. Files in this +// directory aren't removed by the 'go clean -cache' command or by Trim. +// They may be removed with 'go clean -fuzzcache'. +// +// TODO(#48526): make Trim remove unused files from this directory. +func (c *DiskCache) FuzzDir() string { + return filepath.Join(c.dir, "fuzz") +} diff --git a/internal/cache/cache_gcil.go b/internal/go/cache/cache_gcil.go similarity index 100% rename from internal/cache/cache_gcil.go rename to internal/go/cache/cache_gcil.go diff --git a/internal/cache/cache_test.go b/internal/go/cache/cache_test.go similarity index 99% rename from internal/cache/cache_test.go rename to internal/go/cache/cache_test.go index b201cc675051..208abcedc923 100644 --- a/internal/cache/cache_test.go +++ b/internal/go/cache/cache_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/golangci/golangci-lint/internal/testenv" + "github.com/golangci/golangci-lint/internal/go/testenv" ) func init() { diff --git a/internal/cache/default.go b/internal/go/cache/default.go similarity index 100% rename from internal/cache/default.go rename to internal/go/cache/default.go diff --git a/internal/cache/default_gcil.go b/internal/go/cache/default_gcil.go similarity index 100% rename from internal/cache/default_gcil.go rename to internal/go/cache/default_gcil.go diff --git a/internal/cache/hash.go b/internal/go/cache/hash.go similarity index 100% rename from internal/cache/hash.go rename to internal/go/cache/hash.go diff --git a/internal/cache/hash_gcil.go b/internal/go/cache/hash_gcil.go similarity index 100% rename from internal/cache/hash_gcil.go rename to internal/go/cache/hash_gcil.go diff --git a/internal/cache/hash_test.go b/internal/go/cache/hash_test.go similarity index 100% rename from internal/cache/hash_test.go rename to internal/go/cache/hash_test.go diff --git a/internal/cache/prog.go b/internal/go/cache/prog.go similarity index 99% rename from internal/cache/prog.go rename to internal/go/cache/prog.go index 91ef831755a4..a93740a3cf5d 100644 --- a/internal/cache/prog.go +++ b/internal/go/cache/prog.go @@ -21,7 +21,7 @@ import ( "sync/atomic" "time" - "github.com/golangci/golangci-lint/internal/quoted" + "github.com/golangci/golangci-lint/internal/go/quoted" ) // ProgCache implements Cache via JSON messages over stdin/stdout to a child diff --git a/internal/cache/readme.md b/internal/go/cache/readme.md similarity index 70% rename from internal/cache/readme.md rename to internal/go/cache/readme.md index 6a774536a610..5be600e42525 100644 --- a/internal/cache/readme.md +++ b/internal/go/cache/readme.md @@ -2,18 +2,31 @@ Extracted from `go/src/cmd/go/internal/cache/`. -- sync with go1.23.2 -- sync with go1.22.8 -- sync with go1.21.13 -- sync with go1.20.14 -- sync with go1.19.13 -- sync with go1.18.10 -- sync with go1.17.13 -- sync with go1.16.15 -- sync with go1.15.15 -- sync with go1.14.15 +The main modifications are: +- The errors management + - Some methods return error. + - Some errors are returned instead of being ignored. +- The name of the env vars: + - `GOCACHE` -> `GOLANGCI_LINT_CACHE` + - `GOCACHEPROG` -> `GOLANGCI_LINT_CACHEPROG` -# Previous History +## History + +- https://github.com/golangci/golangci-lint/pull/5100 + - Move package from `internal/cache` to `internal/go/cache` +- https://github.com/golangci/golangci-lint/pull/5098 + - sync with go1.23.2 + - sync with go1.22.8 + - sync with go1.21.13 + - sync with go1.20.14 + - sync with go1.19.13 + - sync with go1.18.10 + - sync with go1.17.13 + - sync with go1.16.15 + - sync with go1.15.15 + - sync with go1.14.15 + +## Previous History Based on the initial PR/commit the based in a mix between go1.12 and go1.13: - cache.go (go1.13) diff --git a/internal/mmap/mmap.go b/internal/go/mmap/mmap.go similarity index 100% rename from internal/mmap/mmap.go rename to internal/go/mmap/mmap.go diff --git a/internal/mmap/mmap_other.go b/internal/go/mmap/mmap_other.go similarity index 100% rename from internal/mmap/mmap_other.go rename to internal/go/mmap/mmap_other.go diff --git a/internal/mmap/mmap_unix.go b/internal/go/mmap/mmap_unix.go similarity index 100% rename from internal/mmap/mmap_unix.go rename to internal/go/mmap/mmap_unix.go diff --git a/internal/mmap/mmap_windows.go b/internal/go/mmap/mmap_windows.go similarity index 100% rename from internal/mmap/mmap_windows.go rename to internal/go/mmap/mmap_windows.go diff --git a/internal/go/mmap/readme.md b/internal/go/mmap/readme.md new file mode 100644 index 000000000000..f68aef097c88 --- /dev/null +++ b/internal/go/mmap/readme.md @@ -0,0 +1,15 @@ +# mmap + +Extracted from `go/src/cmd/go/internal/mmap/` (related to `cache`). +This is just a copy of the Go code without any changes. + +## History + +- https://github.com/golangci/golangci-lint/pull/5100 + - Move package from `internal/mmap` to `internal/go/mmap` +- https://github.com/golangci/golangci-lint/pull/5098 + - sync with go1.23.2 + - sync with go1.22.8 + - sync with go1.21.13 + - sync with go1.20.14 + - sync with go1.19.13 diff --git a/internal/quoted/quoted.go b/internal/go/quoted/quoted.go similarity index 100% rename from internal/quoted/quoted.go rename to internal/go/quoted/quoted.go diff --git a/internal/quoted/quoted_test.go b/internal/go/quoted/quoted_test.go similarity index 100% rename from internal/quoted/quoted_test.go rename to internal/go/quoted/quoted_test.go diff --git a/internal/go/quoted/readme.md b/internal/go/quoted/readme.md new file mode 100644 index 000000000000..a5e4c4bb3b09 --- /dev/null +++ b/internal/go/quoted/readme.md @@ -0,0 +1,13 @@ +# quoted + +Extracted from `go/src/cmd/internal/quoted/` (related to `cache`). +This is just a copy of the Go code without any changes. + +## History + +- https://github.com/golangci/golangci-lint/pull/5100 + - Move package from `internal/quoted` to `internal/go/quoted` +- https://github.com/golangci/golangci-lint/pull/5098 + - sync go1.23.2 + - sync go1.22.8 + - sync go1.21.13 diff --git a/internal/robustio/readme.md b/internal/go/robustio/readme.md similarity index 64% rename from internal/robustio/readme.md rename to internal/go/robustio/readme.md index 7c7ba0483ae5..f4dbc162649d 100644 --- a/internal/robustio/readme.md +++ b/internal/go/robustio/readme.md @@ -4,3 +4,8 @@ Extracted from go1.19.1/src/cmd/go/internal/robustio There is only one modification: - ERROR_SHARING_VIOLATION extracted from go1.19.1/src/internal/syscall/windows/syscall_windows.go to remove the dependencies to `internal/syscall/windows` + +## History + +- https://github.com/golangci/golangci-lint/pull/5100 + - Move package from `internal/robustio` to `internal/go/robustio` diff --git a/internal/robustio/robustio.go b/internal/go/robustio/robustio.go similarity index 100% rename from internal/robustio/robustio.go rename to internal/go/robustio/robustio.go diff --git a/internal/robustio/robustio_darwin.go b/internal/go/robustio/robustio_darwin.go similarity index 100% rename from internal/robustio/robustio_darwin.go rename to internal/go/robustio/robustio_darwin.go diff --git a/internal/robustio/robustio_flaky.go b/internal/go/robustio/robustio_flaky.go similarity index 100% rename from internal/robustio/robustio_flaky.go rename to internal/go/robustio/robustio_flaky.go diff --git a/internal/robustio/robustio_other.go b/internal/go/robustio/robustio_other.go similarity index 100% rename from internal/robustio/robustio_other.go rename to internal/go/robustio/robustio_other.go diff --git a/internal/robustio/robustio_windows.go b/internal/go/robustio/robustio_windows.go similarity index 100% rename from internal/robustio/robustio_windows.go rename to internal/go/robustio/robustio_windows.go diff --git a/internal/go/testenv/readme.md b/internal/go/testenv/readme.md new file mode 100644 index 000000000000..6610210ee193 --- /dev/null +++ b/internal/go/testenv/readme.md @@ -0,0 +1,14 @@ +# testenv + +Extracted from `go/src/internal/testenv/`. + +Only the function `SyscallIsNotSupported` is extracted (related to `cache`). + +## History + +- https://github.com/golangci/golangci-lint/pull/5100 + - Move package from `internal/testenv` to `internal/go/testenv` +- https://github.com/golangci/golangci-lint/pull/5098 + - sync with go1.23.2 + - sync with go1.22.8 + - sync with go1.21.13 diff --git a/internal/testenv/testenv.go b/internal/go/testenv/testenv.go similarity index 100% rename from internal/testenv/testenv.go rename to internal/go/testenv/testenv.go diff --git a/internal/testenv/testenv_notunix.go b/internal/go/testenv/testenv_notunix.go similarity index 100% rename from internal/testenv/testenv_notunix.go rename to internal/go/testenv/testenv_notunix.go diff --git a/internal/testenv/testenv_notwin.go b/internal/go/testenv/testenv_notwin.go similarity index 100% rename from internal/testenv/testenv_notwin.go rename to internal/go/testenv/testenv_notwin.go diff --git a/internal/testenv/testenv_unix.go b/internal/go/testenv/testenv_unix.go similarity index 100% rename from internal/testenv/testenv_unix.go rename to internal/go/testenv/testenv_unix.go diff --git a/internal/testenv/testenv_windows.go b/internal/go/testenv/testenv_windows.go similarity index 100% rename from internal/testenv/testenv_windows.go rename to internal/go/testenv/testenv_windows.go diff --git a/internal/mmap/LICENSE b/internal/mmap/LICENSE deleted file mode 100644 index 6a66aea5eafe..000000000000 --- a/internal/mmap/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/mmap/readme.md b/internal/mmap/readme.md deleted file mode 100644 index b5ec2d36840d..000000000000 --- a/internal/mmap/readme.md +++ /dev/null @@ -1,9 +0,0 @@ -# mmap - -Extracted from `go/src/cmd/go/internal/mmap/` (related to `cache`). - -- sync with go1.23.2 -- sync with go1.22.8 -- sync with go1.21.13 -- sync with go1.20.14 -- sync with go1.19.13 diff --git a/internal/pkgcache/pkgcache.go b/internal/pkgcache/pkgcache.go deleted file mode 100644 index 4a66ce5a8387..000000000000 --- a/internal/pkgcache/pkgcache.go +++ /dev/null @@ -1,228 +0,0 @@ -package pkgcache - -import ( - "bytes" - "encoding/gob" - "encoding/hex" - "errors" - "fmt" - "runtime" - "sort" - "sync" - - "golang.org/x/tools/go/packages" - - "github.com/golangci/golangci-lint/internal/cache" - "github.com/golangci/golangci-lint/pkg/logutils" - "github.com/golangci/golangci-lint/pkg/timeutils" -) - -type HashMode int - -const ( - HashModeNeedOnlySelf HashMode = iota - HashModeNeedDirectDeps - HashModeNeedAllDeps -) - -// Cache is a per-package data cache. A cached data is invalidated when -// package, or it's dependencies change. -type Cache struct { - lowLevelCache cache.Cache - pkgHashes sync.Map - sw *timeutils.Stopwatch - log logutils.Log - ioSem chan struct{} // semaphore limiting parallel IO -} - -func NewCache(sw *timeutils.Stopwatch, log logutils.Log) (*Cache, error) { - return &Cache{ - lowLevelCache: cache.Default(), - sw: sw, - log: log, - ioSem: make(chan struct{}, runtime.GOMAXPROCS(-1)), - }, nil -} - -func (c *Cache) Close() { - c.sw.TrackStage("close", func() { - err := c.lowLevelCache.Close() - if err != nil { - c.log.Errorf("cache close: %v", err) - } - }) -} - -func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data any) error { - var err error - buf := &bytes.Buffer{} - c.sw.TrackStage("gob", func() { - err = gob.NewEncoder(buf).Encode(data) - }) - if err != nil { - return fmt.Errorf("failed to gob encode: %w", err) - } - - var aID cache.ActionID - - c.sw.TrackStage("key build", func() { - aID, err = c.pkgActionID(pkg, mode) - if err == nil { - subkey, subkeyErr := cache.Subkey(aID, key) - if subkeyErr != nil { - err = fmt.Errorf("failed to build subkey: %w", subkeyErr) - } - aID = subkey - } - }) - if err != nil { - return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) - } - c.ioSem <- struct{}{} - c.sw.TrackStage("cache io", func() { - err = cache.PutBytes(c.lowLevelCache, aID, buf.Bytes()) - }) - <-c.ioSem - if err != nil { - return fmt.Errorf("failed to save data to low-level cache by key %s for package %s: %w", key, pkg.Name, err) - } - - return nil -} - -var ErrMissing = errors.New("missing data") - -func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data any) error { - var aID cache.ActionID - var err error - c.sw.TrackStage("key build", func() { - aID, err = c.pkgActionID(pkg, mode) - if err == nil { - subkey, subkeyErr := cache.Subkey(aID, key) - if subkeyErr != nil { - err = fmt.Errorf("failed to build subkey: %w", subkeyErr) - } - aID = subkey - } - }) - if err != nil { - return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) - } - - var b []byte - c.ioSem <- struct{}{} - c.sw.TrackStage("cache io", func() { - b, _, err = cache.GetBytes(c.lowLevelCache, aID) - }) - <-c.ioSem - if err != nil { - if cache.IsErrMissing(err) { - return ErrMissing - } - return fmt.Errorf("failed to get data from low-level cache by key %s for package %s: %w", key, pkg.Name, err) - } - - c.sw.TrackStage("gob", func() { - err = gob.NewDecoder(bytes.NewReader(b)).Decode(data) - }) - if err != nil { - return fmt.Errorf("failed to gob decode: %w", err) - } - - return nil -} - -func (c *Cache) pkgActionID(pkg *packages.Package, mode HashMode) (cache.ActionID, error) { - hash, err := c.packageHash(pkg, mode) - if err != nil { - return cache.ActionID{}, fmt.Errorf("failed to get package hash: %w", err) - } - - key, err := cache.NewHash("action ID") - if err != nil { - return cache.ActionID{}, fmt.Errorf("failed to make a hash: %w", err) - } - fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) - fmt.Fprintf(key, "pkghash %s\n", hash) - - return key.Sum(), nil -} - -// packageHash computes a package's hash. The hash is based on all Go -// files that make up the package, as well as the hashes of imported -// packages. -func (c *Cache) packageHash(pkg *packages.Package, mode HashMode) (string, error) { - type hashResults map[HashMode]string - hashResI, ok := c.pkgHashes.Load(pkg) - if ok { - hashRes := hashResI.(hashResults) - if _, ok := hashRes[mode]; !ok { - return "", fmt.Errorf("no mode %d in hash result", mode) - } - return hashRes[mode], nil - } - - hashRes := hashResults{} - - key, err := cache.NewHash("package hash") - if err != nil { - return "", fmt.Errorf("failed to make a hash: %w", err) - } - - fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) - for _, f := range pkg.CompiledGoFiles { - c.ioSem <- struct{}{} - h, fErr := cache.FileHash(f) - <-c.ioSem - if fErr != nil { - return "", fmt.Errorf("failed to calculate file %s hash: %w", f, fErr) - } - fmt.Fprintf(key, "file %s %x\n", f, h) - } - curSum := key.Sum() - hashRes[HashModeNeedOnlySelf] = hex.EncodeToString(curSum[:]) - - imps := make([]*packages.Package, 0, len(pkg.Imports)) - for _, imp := range pkg.Imports { - imps = append(imps, imp) - } - sort.Slice(imps, func(i, j int) bool { - return imps[i].PkgPath < imps[j].PkgPath - }) - - calcDepsHash := func(depMode HashMode) error { - for _, dep := range imps { - if dep.PkgPath == "unsafe" { - continue - } - - depHash, depErr := c.packageHash(dep, depMode) - if depErr != nil { - return fmt.Errorf("failed to calculate hash for dependency %s with mode %d: %w", dep.Name, depMode, depErr) - } - - fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, depHash) - } - return nil - } - - if err := calcDepsHash(HashModeNeedOnlySelf); err != nil { - return "", err - } - - curSum = key.Sum() - hashRes[HashModeNeedDirectDeps] = hex.EncodeToString(curSum[:]) - - if err := calcDepsHash(HashModeNeedAllDeps); err != nil { - return "", err - } - curSum = key.Sum() - hashRes[HashModeNeedAllDeps] = hex.EncodeToString(curSum[:]) - - if _, ok := hashRes[mode]; !ok { - return "", fmt.Errorf("invalid mode %d", mode) - } - - c.pkgHashes.Store(pkg, hashRes) - return hashRes[mode], nil -} diff --git a/internal/quoted/LICENSE b/internal/quoted/LICENSE deleted file mode 100644 index 6a66aea5eafe..000000000000 --- a/internal/quoted/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/quoted/readme.md b/internal/quoted/readme.md deleted file mode 100644 index a3c33d624513..000000000000 --- a/internal/quoted/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# quoted - -Extracted from `go/src/cmd/internal/quoted/` (related to `cache`). - -- sync go1.23.2 -- sync go1.22.8 -- sync go1.21.13 diff --git a/internal/robustio/LICENSE b/internal/robustio/LICENSE deleted file mode 100644 index 6a66aea5eafe..000000000000 --- a/internal/robustio/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/testenv/LICENSE b/internal/testenv/LICENSE deleted file mode 100644 index 6a66aea5eafe..000000000000 --- a/internal/testenv/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/testenv/readme.md b/internal/testenv/readme.md deleted file mode 100644 index 528890c26fa6..000000000000 --- a/internal/testenv/readme.md +++ /dev/null @@ -1,9 +0,0 @@ -# testenv - -Extracted from `go/src/internal/testenv/`. - -Only the function `SyscallIsNotSupported` is extracted (related to `cache`). - -- sync with go1.23.2 -- sync with go1.22.8 -- sync with go1.21.13 diff --git a/pkg/commands/cache.go b/pkg/commands/cache.go index 0af8cc2856e5..4f2c812dce91 100644 --- a/pkg/commands/cache.go +++ b/pkg/commands/cache.go @@ -51,7 +51,7 @@ func newCacheCommand() *cacheCommand { } func (*cacheCommand) executeClean(_ *cobra.Command, _ []string) error { - cacheDir, _ := cache.DefaultDir() + cacheDir := cache.DefaultDir() if err := os.RemoveAll(cacheDir); err != nil { return fmt.Errorf("failed to remove dir %s: %w", cacheDir, err) @@ -61,7 +61,7 @@ func (*cacheCommand) executeClean(_ *cobra.Command, _ []string) error { } func (*cacheCommand) executeStatus(_ *cobra.Command, _ []string) { - cacheDir, _ := cache.DefaultDir() + cacheDir := cache.DefaultDir() _, _ = fmt.Fprintf(logutils.StdOut, "Dir: %s\n", cacheDir) diff --git a/pkg/commands/run.go b/pkg/commands/run.go index f289bfdd79d5..ff7c5e467b67 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -28,7 +28,6 @@ import ( "gopkg.in/yaml.v3" "github.com/golangci/golangci-lint/internal/cache" - "github.com/golangci/golangci-lint/internal/pkgcache" "github.com/golangci/golangci-lint/pkg/config" "github.com/golangci/golangci-lint/pkg/exitcodes" "github.com/golangci/golangci-lint/pkg/fsutils" @@ -209,7 +208,7 @@ func (c *runCommand) preRunE(_ *cobra.Command, args []string) error { sw := timeutils.NewStopwatch("pkgcache", c.log.Child(logutils.DebugKeyStopwatch)) - pkgCache, err := pkgcache.NewCache(sw, c.log.Child(logutils.DebugKeyPkgCache)) + pkgCache, err := cache.NewCache(sw, c.log.Child(logutils.DebugKeyPkgCache)) if err != nil { return fmt.Errorf("failed to build packages cache: %w", err) } @@ -640,7 +639,7 @@ func initHashSalt(version string, cfg *config.Config) error { b := bytes.NewBuffer(binSalt) b.Write(configSalt) - cache.SetSalt(b.Bytes()) + cache.SetSalt(b) return nil } diff --git a/pkg/goanalysis/runner.go b/pkg/goanalysis/runner.go index 806aac81064c..dfcdcbaaa4d5 100644 --- a/pkg/goanalysis/runner.go +++ b/pkg/goanalysis/runner.go @@ -21,8 +21,8 @@ import ( "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/packages" + "github.com/golangci/golangci-lint/internal/cache" "github.com/golangci/golangci-lint/internal/errorutil" - "github.com/golangci/golangci-lint/internal/pkgcache" "github.com/golangci/golangci-lint/pkg/goanalysis/load" "github.com/golangci/golangci-lint/pkg/logutils" "github.com/golangci/golangci-lint/pkg/timeutils" @@ -52,7 +52,7 @@ type Diagnostic struct { type runner struct { log logutils.Log prefix string // ensure unique analyzer names - pkgCache *pkgcache.Cache + pkgCache *cache.Cache loadGuard *load.Guard loadMode LoadMode passToPkg map[*analysis.Pass]*packages.Package @@ -60,7 +60,7 @@ type runner struct { sw *timeutils.Stopwatch } -func newRunner(prefix string, logger logutils.Log, pkgCache *pkgcache.Cache, loadGuard *load.Guard, +func newRunner(prefix string, logger logutils.Log, pkgCache *cache.Cache, loadGuard *load.Guard, loadMode LoadMode, sw *timeutils.Stopwatch, ) *runner { return &runner{ diff --git a/pkg/goanalysis/runner_action.go b/pkg/goanalysis/runner_action.go index 58ea297ea92b..81e2556b39f9 100644 --- a/pkg/goanalysis/runner_action.go +++ b/pkg/goanalysis/runner_action.go @@ -13,8 +13,8 @@ import ( "golang.org/x/tools/go/packages" "golang.org/x/tools/go/types/objectpath" + "github.com/golangci/golangci-lint/internal/cache" "github.com/golangci/golangci-lint/internal/errorutil" - "github.com/golangci/golangci-lint/internal/pkgcache" "github.com/golangci/golangci-lint/pkg/goanalysis/pkgerrors" ) @@ -328,14 +328,14 @@ func (act *action) persistFactsToCache() error { factsCacheDebugf("Caching %d facts for package %q and analyzer %s", len(facts), act.pkg.Name, act.a.Name) key := fmt.Sprintf("%s/facts", analyzer.Name) - return act.r.pkgCache.Put(act.pkg, pkgcache.HashModeNeedAllDeps, key, facts) + return act.r.pkgCache.Put(act.pkg, cache.HashModeNeedAllDeps, key, facts) } func (act *action) loadPersistedFacts() bool { var facts []Fact key := fmt.Sprintf("%s/facts", act.a.Name) - if err := act.r.pkgCache.Get(act.pkg, pkgcache.HashModeNeedAllDeps, key, &facts); err != nil { - if !errors.Is(err, pkgcache.ErrMissing) && !errors.Is(err, io.EOF) { + if err := act.r.pkgCache.Get(act.pkg, cache.HashModeNeedAllDeps, key, &facts); err != nil { + if !errors.Is(err, cache.ErrMissing) && !errors.Is(err, io.EOF) { act.r.log.Warnf("Failed to get persisted facts: %s", err) } diff --git a/pkg/goanalysis/runners.go b/pkg/goanalysis/runners.go index 698439f08196..033cba81d5de 100644 --- a/pkg/goanalysis/runners.go +++ b/pkg/goanalysis/runners.go @@ -12,7 +12,7 @@ import ( "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/packages" - "github.com/golangci/golangci-lint/internal/pkgcache" + "github.com/golangci/golangci-lint/internal/cache" "github.com/golangci/golangci-lint/pkg/goanalysis/pkgerrors" "github.com/golangci/golangci-lint/pkg/lint/linter" "github.com/golangci/golangci-lint/pkg/logutils" @@ -163,7 +163,7 @@ func saveIssuesToCache(allPkgs []*packages.Package, pkgsFromCache map[*packages. } atomic.AddInt64(&savedIssuesCount, int64(len(encodedIssues))) - if err := lintCtx.PkgCache.Put(pkg, pkgcache.HashModeNeedAllDeps, lintResKey, encodedIssues); err != nil { + if err := lintCtx.PkgCache.Put(pkg, cache.HashModeNeedAllDeps, lintResKey, encodedIssues); err != nil { lintCtx.Log.Infof("Failed to save package %s issues (%d) to cache: %s", pkg, len(pkgIssues), err) } else { issuesCacheDebugf("Saved package %s issues (%d) to cache", pkg, len(pkgIssues)) @@ -212,7 +212,7 @@ func loadIssuesFromCache(pkgs []*packages.Package, lintCtx *linter.Context, defer wg.Done() for pkg := range pkgCh { var pkgIssues []EncodingIssue - err := lintCtx.PkgCache.Get(pkg, pkgcache.HashModeNeedAllDeps, lintResKey, &pkgIssues) + err := lintCtx.PkgCache.Get(pkg, cache.HashModeNeedAllDeps, lintResKey, &pkgIssues) cacheRes := pkgToCacheRes[pkg] cacheRes.loadErr = err if err != nil { diff --git a/pkg/lint/context.go b/pkg/lint/context.go index 160620338f8c..d04a11b81f1a 100644 --- a/pkg/lint/context.go +++ b/pkg/lint/context.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/golangci/golangci-lint/internal/pkgcache" + "github.com/golangci/golangci-lint/internal/cache" "github.com/golangci/golangci-lint/pkg/config" "github.com/golangci/golangci-lint/pkg/exitcodes" "github.com/golangci/golangci-lint/pkg/fsutils" @@ -19,13 +19,13 @@ type ContextBuilder struct { pkgLoader *PackageLoader fileCache *fsutils.FileCache - pkgCache *pkgcache.Cache + pkgCache *cache.Cache loadGuard *load.Guard } func NewContextBuilder(cfg *config.Config, pkgLoader *PackageLoader, - fileCache *fsutils.FileCache, pkgCache *pkgcache.Cache, loadGuard *load.Guard, + fileCache *fsutils.FileCache, pkgCache *cache.Cache, loadGuard *load.Guard, ) *ContextBuilder { return &ContextBuilder{ cfg: cfg, diff --git a/pkg/lint/linter/context.go b/pkg/lint/linter/context.go index 5c03630b26df..9f29b5c4c884 100644 --- a/pkg/lint/linter/context.go +++ b/pkg/lint/linter/context.go @@ -5,7 +5,7 @@ import ( "golang.org/x/tools/go/packages" - "github.com/golangci/golangci-lint/internal/pkgcache" + "github.com/golangci/golangci-lint/internal/cache" "github.com/golangci/golangci-lint/pkg/config" "github.com/golangci/golangci-lint/pkg/fsutils" "github.com/golangci/golangci-lint/pkg/goanalysis/load" @@ -24,7 +24,7 @@ type Context struct { FileCache *fsutils.FileCache Log logutils.Log - PkgCache *pkgcache.Cache + PkgCache *cache.Cache LoadGuard *load.Guard } diff --git a/pkg/result/processors/fixer.go b/pkg/result/processors/fixer.go index 4915dc479a45..cc99e6940ef8 100644 --- a/pkg/result/processors/fixer.go +++ b/pkg/result/processors/fixer.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "github.com/golangci/golangci-lint/internal/robustio" + "github.com/golangci/golangci-lint/internal/go/robustio" "github.com/golangci/golangci-lint/pkg/config" "github.com/golangci/golangci-lint/pkg/fsutils" "github.com/golangci/golangci-lint/pkg/logutils"