dep: Introduce noverify field to Gopkg.toml

This field allows bypassing of vendor verification on a
project-by-project basis. When set, `dep check` will ignore any hash
verification problems with the directory, and `dep ensure` will not
rewrite the dependency unless it is absent from vendor entirely, pruning
rules change, or solving changes one of the other, higher-order
properties.

Author: sdboyer

cmd/dep/check.go

@@ -132,18 +132,32 @@ func (cmd *checkCommand) Run(ctx *dep.Ctx, args []string) error {
 			logger.Println()
 		}
 
+		noverify := make(map[string]bool)
+		for _, skip := range p.Manifest.NoVerify {
+			noverify[skip] = true
+		}
+
 		var vendorfail bool
 		// One full pass through, to see if we need to print the header, and to
 		// create an array of names to sort for deterministic output.
 		var ordered []string
 		for path, status := range statuses {
 			ordered = append(ordered, path)
-			if status != verify.NoMismatch {
+
+			switch status {
+			case verify.DigestMismatchInLock, verify.HashVersionMismatch, verify.EmptyDigestInLock:
+				// NoVerify applies only to these three cases.
+				if noverify[path] {
+					continue
+				}
+				fallthrough
+			case verify.NotInTree, verify.NotInLock:
 				fail = true
 				if !vendorfail {
 					vendorfail = true
 					logger.Println("# vendor is out of sync:")
 				}
+
 			}
 		}
 		sort.Strings(ordered)
@@ -158,19 +172,26 @@ func (cmd *checkCommand) Run(ctx *dep.Ctx, args []string) error {
 				if err != nil {
 					return errors.Wrap(err, "could not stat file that VerifyVendor claimed existed")
 				}
-
 				if fi.IsDir() {
 					logger.Printf("%s: unused project\n", pr)
 				} else {
 					logger.Printf("%s: orphaned file\n", pr)
 				}
-			case verify.DigestMismatchInLock:
-				logger.Printf("%s: hash of vendored tree didn't match digest in Gopkg.lock\n", pr)
-			case verify.HashVersionMismatch:
-				// This will double-print if the hash version is zero, but
-				// that's a rare case that really only occurs before the first
-				// run with a version of dep >=0.5.0, so it's fine.
-				logger.Printf("%s: hash algorithm mismatch, want version %v\n", pr, verify.HashVersion)
+			case verify.DigestMismatchInLock, verify.HashVersionMismatch, verify.EmptyDigestInLock:
+				// NoVerify applies only to these three cases.
+				if !noverify[pr] {
+					switch status {
+					case verify.DigestMismatchInLock:
+						logger.Printf("%s: hash of vendored tree not equal to digest in Gopkg.lock\n", pr)
+					case verify.EmptyDigestInLock:
+						logger.Printf("%s: no digest in Gopkg.lock to compare against hash of vendored tree\n", pr)
+					case verify.HashVersionMismatch:
+						// This will double-print if the hash version is zero, but
+						// that's a rare case that really only occurs before the first
+						// run with a version of dep >=0.5.0, so it's fine.
+						logger.Printf("%s: hash algorithm mismatch, want version %v\n", pr, verify.HashVersion)
+					}
+				}
 			}
 		}
 	}
@@ -185,12 +206,12 @@ func sprintLockUnsat(lsat verify.LockSatisfaction) string {
 	var buf bytes.Buffer
 	sort.Strings(lsat.MissingImports)
 	for _, missing := range lsat.MissingImports {
-		fmt.Fprintf(&buf, "%s: missing from input-imports\n", missing)
+		fmt.Fprintf(&buf, "%s: imported or required, but missing from Gopkg.lock's input-imports\n", missing)
 	}
 
 	sort.Strings(lsat.ExcessImports)
 	for _, excess := range lsat.ExcessImports {
-		fmt.Fprintf(&buf, "%s: in input-imports, but not imported\n", excess)
+		fmt.Fprintf(&buf, "%s: in Gopkg.lock's input-imports, but neither imported nor required\n", excess)
 	}
 
 	var ordered []string

cmd/dep/ensure.go

@@ -285,11 +285,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
 		lock = dep.LockFromSolution(solution, p.Manifest.PruneOptions)
 	}
 
-	status, err := p.VerifyVendor()
-	if err != nil {
-		return errors.Wrap(err, "error while verifying vendor directory")
-	}
-	dw, err := dep.NewDeltaWriter(p.Lock, lock, status, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"), cmd.vendorBehavior())
+	dw, err := dep.NewDeltaWriter(p, lock, cmd.vendorBehavior())
 	if err != nil {
 		return err
 	}
@@ -365,11 +361,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
 		return handleAllTheFailuresOfTheWorld(err)
 	}
 
-	status, err := p.VerifyVendor()
-	if err != nil {
-		return errors.Wrap(err, "error while verifying vendor directory")
-	}
-	dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), status, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"), cmd.vendorBehavior())
+	dw, err := dep.NewDeltaWriter(p, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior())
 	if err != nil {
 		return err
 	}
@@ -411,11 +403,6 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
 		}
 	}
 
-	//exrmap, err := p.GetDirectDependencyNames(sm)
-	//if err != nil {
-	//return err
-	//}
-
 	// Note: these flags are only partially used by the latter parts of the
 	// algorithm; rather, it relies on inference. However, they remain in their
 	// entirety as future needs may make further use of them, being a handy,
@@ -643,11 +630,7 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
 	}
 	sort.Strings(reqlist)
 
-	status, err := p.VerifyVendor()
-	if err != nil {
-		return errors.Wrap(err, "error while verifying vendor directory")
-	}
-	dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), status, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"), cmd.vendorBehavior())
+	dw, err := dep.NewDeltaWriter(p, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior())
 	if err != nil {
 		return err
 	}

manifest.go

@@ -28,6 +28,7 @@ var (
 	errInvalidOverride     = errors.Errorf("%q must be a TOML array of tables", "override")
 	errInvalidRequired     = errors.Errorf("%q must be a TOML list of strings", "required")
 	errInvalidIgnored      = errors.Errorf("%q must be a TOML list of strings", "ignored")
+	errInvalidNoVerify     = errors.Errorf("%q must be a TOML list of strings", "noverify")
 	errInvalidPrune        = errors.Errorf("%q must be a TOML table of booleans", "prune")
 	errInvalidPruneProject = errors.Errorf("%q must be a TOML array of tables", "prune.project")
 	errInvalidMetadata     = errors.New("metadata should be a TOML table")
@@ -51,6 +52,8 @@ type Manifest struct {
 	Ignored  []string
 	Required []string
 
+	NoVerify []string
+
 	PruneOptions gps.CascadingPruneOptions
 }
 
@@ -59,6 +62,7 @@ type rawManifest struct {
 	Overrides    []rawProject    `toml:"override,omitempty"`
 	Ignored      []string        `toml:"ignored,omitempty"`
 	Required     []string        `toml:"required,omitempty"`
+	NoVerify     []string        `toml:"noverify,omitempty"`
 	PruneOptions rawPruneOptions `toml:"prune,omitempty"`
 }
 
@@ -85,7 +89,7 @@ const (
 	pruneOptionNonGo          = "non-go"
 )
 
-// Constants to represents per-project prune uint8 values.
+// Constants representing per-project prune uint8 values.
 const (
 	pvnone  uint8 = 0 // No per-project prune value was set in Gopkg.toml.
 	pvtrue  uint8 = 1 // Per-project prune value was explicitly set to true.
@@ -182,7 +186,7 @@ func validateManifest(s string) ([]error, error) {
 					return warns, errInvalidOverride
 				}
 			}
-		case "ignored", "required":
+		case "ignored", "required", "noverify":
 			valid := true
 			if rawList, ok := val.([]interface{}); ok {
 				// Check element type of the array. TOML doesn't let mixing of types in
@@ -201,6 +205,9 @@ func validateManifest(s string) ([]error, error) {
 				if prop == "required" {
 					return warns, errInvalidRequired
 				}
+				if prop == "noverify" {
+					return warns, errInvalidNoVerify
+				}
 			}
 		case "prune":
 			pruneWarns, err := validatePruneOptions(val, true)
@@ -368,6 +375,7 @@ func fromRawManifest(raw rawManifest, buf *bytes.Buffer) (*Manifest, error) {
 	m.Ovr = make(gps.ProjectConstraints, len(raw.Overrides))
 	m.Ignored = raw.Ignored
 	m.Required = raw.Required
+	m.NoVerify = raw.NoVerify
 
 	for i := 0; i < len(raw.Constraints); i++ {
 		name, prj, err := toProject(raw.Constraints[i])
@@ -532,6 +540,7 @@ func (m *Manifest) toRaw() rawManifest {
 		Overrides:   make([]rawProject, 0, len(m.Ovr)),
 		Ignored:     m.Ignored,
 		Required:    m.Required,
+		NoVerify:    m.NoVerify,
 	}
 
 	for n, prj := range m.Constraints {

txn_writer.go

@@ -423,10 +423,11 @@ type DeltaWriter struct {
 type changeType uint8
 
 const (
-	solveChanged changeType = iota + 1
-	hashMismatch
+	hashMismatch changeType = iota + 1
 	hashVersionMismatch
 	hashAbsent
+	noVerify
+	solveChanged
 	pruneOptsChanged
 	missingFromTree
 	projectAdded
@@ -437,10 +438,10 @@ const (
 // directory by writing out only those projects that actually need to be written
 // out - they have changed in some way, or they lack the necessary hash
 // information to be verified.
-func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatus, prune gps.CascadingPruneOptions, vendorDir string, behavior VendorBehavior) (TreeWriter, error) {
-	sw := &DeltaWriter{
+func NewDeltaWriter(p *Project, newLock *Lock, behavior VendorBehavior) (TreeWriter, error) {
+	dw := &DeltaWriter{
 		lock:      newLock,
-		vendorDir: vendorDir,
+		vendorDir: filepath.Join(p.AbsRoot, "vendor"),
 		changed:   make(map[gps.ProjectRoot]changeType),
 		behavior:  behavior,
 	}
@@ -449,27 +450,35 @@ func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatu
 		return nil, errors.New("must provide a non-nil newlock")
 	}
 
-	_, err := os.Stat(vendorDir)
-	if err != nil && os.IsNotExist(err) {
-		// Provided dir does not exist, so there's no disk contents to compare
-		// against. Fall back to the old SafeWriter.
-		return NewSafeWriter(nil, oldLock, newLock, behavior, prune, status)
+	status, err := p.VerifyVendor()
+	if err != nil {
+		return nil, err
 	}
 
-	sw.lockDiff = verify.DiffLocks(oldLock, newLock)
+	_, err = os.Stat(dw.vendorDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Provided dir does not exist, so there's no disk contents to compare
+			// against. Fall back to the old SafeWriter.
+			return NewSafeWriter(nil, p.Lock, newLock, behavior, p.Manifest.PruneOptions, status)
+		}
+		return nil, err
+	}
 
-	for pr, lpd := range sw.lockDiff.ProjectDeltas {
+	dw.lockDiff = verify.DiffLocks(p.Lock, newLock)
+
+	for pr, lpd := range dw.lockDiff.ProjectDeltas {
 		// Hash changes aren't relevant at this point, as they could be empty
 		// in the new lock, and therefore a symptom of a solver change.
 		if lpd.Changed(anyExceptHash) {
 			if lpd.WasAdded() {
-				sw.changed[pr] = projectAdded
+				dw.changed[pr] = projectAdded
 			} else if lpd.WasRemoved() {
-				sw.changed[pr] = projectRemoved
+				dw.changed[pr] = projectRemoved
 			} else if lpd.PruneOptsChanged() {
-				sw.changed[pr] = pruneOptsChanged
+				dw.changed[pr] = pruneOptsChanged
 			} else {
-				sw.changed[pr] = solveChanged
+				dw.changed[pr] = solveChanged
 			}
 		}
 	}
@@ -478,23 +487,44 @@ func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatu
 		pr := gps.ProjectRoot(spr)
 		// These cases only matter if there was no change already recorded via
 		// the differ.
-		if _, has := sw.changed[pr]; !has {
+		if _, has := dw.changed[pr]; !has {
 			switch stat {
 			case verify.NotInTree:
-				sw.changed[pr] = missingFromTree
+				dw.changed[pr] = missingFromTree
 			case verify.NotInLock:
-				sw.changed[pr] = projectRemoved
+				dw.changed[pr] = projectRemoved
 			case verify.DigestMismatchInLock:
-				sw.changed[pr] = hashMismatch
+				dw.changed[pr] = hashMismatch
 			case verify.HashVersionMismatch:
-				sw.changed[pr] = hashVersionMismatch
+				dw.changed[pr] = hashVersionMismatch
 			case verify.EmptyDigestInLock:
-				sw.changed[pr] = hashAbsent
+				dw.changed[pr] = hashAbsent
 			}
 		}
 	}
 
-	return sw, nil
+	// Apply noverify last, as it should only supersede changeTypes with lower
+	// values. It is NOT applied if no existing change is registered.
+	for _, spr := range p.Manifest.NoVerify {
+		pr := gps.ProjectRoot(spr)
+		// We don't validate this field elsewhere as it can be difficult to know
+		// at the beginning of a dep ensure command whether or not the noverify
+		// project actually will exist as part of the Lock by the end of the
+		// run. So, only apply if it's in the lockdiff, and isn't a removal.
+		if _, has := dw.lockDiff.ProjectDeltas[pr]; has {
+			if typ, has := dw.changed[pr]; has && typ < noVerify {
+				// Avoid writing noverify projects at all for the lower change
+				// types.
+				delete(dw.changed, pr)
+
+				// Uncomment this if we want to switch to the safer behavior,
+				// where we ALWAYS write noverify projects.
+				//dw.changed[pr] = noVerify
+			}
+		}
+	}
+
+	return dw, nil
 }
 
 // Write executes the planned changes.
@@ -659,6 +689,8 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l
 // possible changeType.
 func changeExplanation(c changeType, lpd verify.LockedProjectDelta) string {
 	switch c {
+	case noVerify:
+		return "verification is disabled"
 	case solveChanged:
 		if lpd.SourceChanged() {
 			return fmt.Sprintf("source changed (%s -> %s)", lpd.SourceBefore, lpd.SourceAfter)
@@ -675,9 +707,8 @@ func changeExplanation(c changeType, lpd verify.LockedProjectDelta) string {
 				return fmt.Sprintf("packages changed (%v added, %v removed)", la, lr)
 			} else if la > 0 {
 				return fmt.Sprintf("packages changed (%v added)", la)
-			} else {
-				return fmt.Sprintf("packages changed (%v removed)", lr)
 			}
+			return fmt.Sprintf("packages changed (%v removed)", lr)
 		}
 	case pruneOptsChanged:
 		// Override what's on the lockdiff with the extra info we have;