cmd/go/internal/lockedfile: add package and support library

lockedfile.File passes through to os.File, with Open, Create, and OpenFile
functions that mimic the corresponding os functions but acquire locks
automatically, releasing them when the file is closed.

lockedfile.Sentinel is a simplified wrapper around lockedfile.OpenFile for the
common use-case of files that signal the status of idempotent tasks.

lockedfile.Mutex is a Mutex-like synchronization primitive implemented in terms
of file locks.

lockedfile.Read is like ioutil.Read, but obtains a read-lock.

lockedfile.Write is like ioutil.Write, but obtains a write-lock and can be used
for read-only files with idempotent contents.

Updates #26794

Change-Id: I50f7132c71d2727862eed54411f3f27e1af55cad
Reviewed-on: https://go-review.googlesource.com/c/145178
Run-TryBot: Bryan C. Mills <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Russ Cox <[email protected]>

Author: Bryan C. Mills

src/cmd/go/internal/lockedfile/internal/filelock/filelock.go

@@ -0,0 +1,98 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package filelock provides a platform-independent API for advisory file
+// locking. Calls to functions in this package on platforms that do not support
+// advisory locks will return errors for which IsNotSupported returns true.
+package filelock
+
+import (
+	"errors"
+	"os"
+)
+
+// A File provides the minimal set of methods required to lock an open file.
+// File implementations must be usable as map keys.
+// The usual implementation is *os.File.
+type File interface {
+	// Name returns the name of the file.
+	Name() string
+
+	// Fd returns a valid file descriptor.
+	// (If the File is an *os.File, it must not be closed.)
+	Fd() uintptr
+
+	// Stat returns the FileInfo structure describing file.
+	Stat() (os.FileInfo, error)
+}
+
+// Lock places an advisory write lock on the file, blocking until it can be
+// locked.
+//
+// If Lock returns nil, no other process will be able to place a read or write
+// lock on the file until this process exits, closes f, or calls Unlock on it.
+//
+// If f's descriptor is already read- or write-locked, the behavior of Lock is
+// unspecified.
+//
+// Closing the file may or may not release the lock promptly. Callers should
+// ensure that Unlock is always called when Lock succeeds.
+func Lock(f File) error {
+	return lock(f, writeLock)
+}
+
+// RLock places an advisory read lock on the file, blocking until it can be locked.
+//
+// If RLock returns nil, no other process will be able to place a write lock on
+// the file until this process exits, closes f, or calls Unlock on it.
+//
+// If f is already read- or write-locked, the behavior of RLock is unspecified.
+//
+// Closing the file may or may not release the lock promptly. Callers should
+// ensure that Unlock is always called if RLock succeeds.
+func RLock(f File) error {
+	return lock(f, readLock)
+}
+
+// Unlock removes an advisory lock placed on f by this process.
+//
+// The caller must not attempt to unlock a file that is not locked.
+func Unlock(f File) error {
+	return unlock(f)
+}
+
+// String returns the name of the function corresponding to lt
+// (Lock, RLock, or Unlock).
+func (lt lockType) String() string {
+	switch lt {
+	case readLock:
+		return "RLock"
+	case writeLock:
+		return "Lock"
+	default:
+		return "Unlock"
+	}
+}
+
+// IsNotSupported returns a boolean indicating whether the error is known to
+// report that a function is not supported (possibly for a specific input).
+// It is satisfied by ErrNotSupported as well as some syscall errors.
+func IsNotSupported(err error) bool {
+	return isNotSupported(underlyingError(err))
+}
+
+var ErrNotSupported = errors.New("operation not supported")
+
+// underlyingError returns the underlying error for known os error types.
+func underlyingError(err error) error {
+	switch err := err.(type) {
+	case *os.PathError:
+		return err.Err
+	case *os.LinkError:
+		return err.Err
+	case *os.SyscallError:
+		return err.Err
+	}
+	return err
+}

src/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!plan9,!solaris,!windows
+
+package filelock
+
+import "os"
+
+type lockType int8
+
+const (
+	readLock = iota + 1
+	writeLock
+)
+
+func lock(f File, lt lockType) error {
+	return &os.PathError{
+		Op:   lt.String(),
+		Path: f.Name(),
+		Err:  ErrNotSupported,
+	}
+}
+
+func unlock(f File) error {
+	return &os.PathError{
+		Op:   "Unlock",
+		Path: f.Name(),
+		Err:  ErrNotSupported,
+	}
+}
+
+func isNotSupported(err error) bool {
+	return err == ErrNotSupported
+}

src/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go

@@ -0,0 +1,38 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build plan9
+
+package filelock
+
+import (
+	"os"
+)
+
+type lockType int8
+
+const (
+	readLock = iota + 1
+	writeLock
+)
+
+func lock(f File, lt lockType) error {
+	return &os.PathError{
+		Op:   lt.String(),
+		Path: f.Name(),
+		Err:  ErrNotSupported,
+	}
+}
+
+func unlock(f File) error {
+	return &os.PathError{
+		Op:   "Unlock",
+		Path: f.Name(),
+		Err:  ErrNotSupported,
+	}
+}
+
+func isNotSupported(err error) bool {
+	return err == ErrNotSupported
+}

src/cmd/go/internal/lockedfile/internal/filelock/filelock_solaris.go

@@ -0,0 +1,157 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code implements the filelock API using POSIX 'fcntl' locks, which attach
+// to an (inode, process) pair rather than a file descriptor. To avoid unlocking
+// files prematurely when the same file is opened through different descriptors,
+// we allow only one read-lock at a time.
+//
+// Most platforms provide some alternative API, such as an 'flock' system call
+// or an F_OFD_SETLK command for 'fcntl', that allows for better concurrency and
+// does not require per-inode bookkeeping in the application.
+//
+// TODO(bcmills): If we add a build tag for Illumos (see golang.org/issue/20603)
+// then Illumos should use F_OFD_SETLK, and the resulting code would be as
+// simple as filelock_unix.go. We will still need the code in this file as long
+// as Oracle Solaris provides only F_SETLK.
+
+package filelock
+
+import (
+	"errors"
+	"io"
+	"os"
+	"sync"
+	"syscall"
+)
+
+type lockType int16
+
+const (
+	readLock  lockType = syscall.F_RDLCK
+	writeLock lockType = syscall.F_WRLCK
+)
+
+type inode = uint64 // type of syscall.Stat_t.Ino
+
+type inodeLock struct {
+	owner File
+	queue []<-chan File
+}
+
+type token struct{}
+
+var (
+	mu     sync.Mutex
+	inodes = map[File]inode{}
+	locks  = map[inode]inodeLock{}
+)
+
+func lock(f File, lt lockType) (err error) {
+	// POSIX locks apply per inode and process, and the lock for an inode is
+	// released when *any* descriptor for that inode is closed. So we need to
+	// synchronize access to each inode internally, and must serialize lock and
+	// unlock calls that refer to the same inode through different descriptors.
+	fi, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	ino := fi.Sys().(*syscall.Stat_t).Ino
+
+	mu.Lock()
+	if i, dup := inodes[f]; dup && i != ino {
+		mu.Unlock()
+		return &os.PathError{
+			Op:   lt.String(),
+			Path: f.Name(),
+			Err:  errors.New("inode for file changed since last Lock or RLock"),
+		}
+	}
+	inodes[f] = ino
+
+	var wait chan File
+	l := locks[ino]
+	if l.owner == f {
+		// This file already owns the lock, but the call may change its lock type.
+	} else if l.owner == nil {
+		// No owner: it's ours now.
+		l.owner = f
+	} else {
+		// Already owned: add a channel to wait on.
+		wait = make(chan File)
+		l.queue = append(l.queue, wait)
+	}
+	locks[ino] = l
+	mu.Unlock()
+
+	if wait != nil {
+		wait <- f
+	}
+
+	err = setlkw(f.Fd(), lt)
+
+	if err != nil {
+		unlock(f)
+		return &os.PathError{
+			Op:   lt.String(),
+			Path: f.Name(),
+			Err:  err,
+		}
+	}
+
+	return nil
+}
+
+func unlock(f File) error {
+	var owner File
+
+	mu.Lock()
+	ino, ok := inodes[f]
+	if ok {
+		owner = locks[ino].owner
+	}
+	mu.Unlock()
+
+	if owner != f {
+		panic("unlock called on a file that is not locked")
+	}
+
+	err := setlkw(f.Fd(), syscall.F_UNLCK)
+
+	mu.Lock()
+	l := locks[ino]
+	if len(l.queue) == 0 {
+		// No waiters: remove the map entry.
+		delete(locks, ino)
+	} else {
+		// The first waiter is sending us their file now.
+		// Receive it and update the queue.
+		l.owner = <-l.queue[0]
+		l.queue = l.queue[1:]
+		locks[ino] = l
+	}
+	delete(inodes, f)
+	mu.Unlock()
+
+	return err
+}
+
+// setlkw calls FcntlFlock with F_SETLKW for the entire file indicated by fd.
+func setlkw(fd uintptr, lt lockType) error {
+	for {
+		err := syscall.FcntlFlock(fd, syscall.F_SETLKW, &syscall.Flock_t{
+			Type:   int16(lt),
+			Whence: io.SeekStart,
+			Start:  0,
+			Len:    0, // All bytes.
+		})
+		if err != syscall.EINTR {
+			return err
+		}
+	}
+}
+
+func isNotSupported(err error) bool {
+	return err == syscall.ENOSYS || err == syscall.ENOTSUP || err == syscall.EOPNOTSUPP || err == ErrNotSupported
+}