dep: Tell the user why we're solving

Add output to all of the information we assemble when checking if the
Lock satisfies the current input set.

Also some refactoring of the ctx.LoadProject() process to have fewer
partial states.

Author: sdboyer

Gopkg.lock

@@ -211,11 +211,6 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error {
 		statchan <- status
 	}(filepath.Join(p.AbsRoot, "vendor"), lps)
 
-	params.RootPackageTree, err = p.ParseRootPackageTree()
-	if err != nil {
-		return err
-	}
-
 	if fatal, err := checkErrors(params.RootPackageTree.Packages, p.Manifest.IgnoredPackages()); err != nil {
 		if fatal {
 			return err
@@ -283,20 +278,32 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
 		return err
 	}
 
-	lock := p.Lock
+	lock := p.ChangedLock
 	if lock != nil {
-		lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree)
+		lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree)
 		if !lsat.Passed() {
-			// TODO(sdboyer) print out what bits are unsatisfied here
+			if ctx.Verbose {
+				ctx.Out.Println("Gopkg.lock is out of sync with Gopkg.toml and project code:")
+				for _, missing := range lsat.MissingImports() {
+					ctx.Out.Printf("\t%s is missing from input-imports\n", missing)
+				}
+				for _, excess := range lsat.ExcessImports() {
+					ctx.Out.Printf("\t%s is in input-imports, but isn't imported\n", excess)
+				}
+				for pr, unmatched := range lsat.UnmatchedOverrides() {
+					ctx.Out.Printf("\t%s is at %s, which is not allowed by override %s\n", pr, unmatched.V, unmatched.C)
+				}
+				for pr, unmatched := range lsat.UnmatchedConstraints() {
+					ctx.Out.Printf("\t%s is at %s, which is not allowed by constraint %s\n", pr, unmatched.V, unmatched.C)
+				}
+				ctx.Out.Println()
+			}
+
 			solver, err := gps.Prepare(params, sm)
 			if err != nil {
 				return errors.Wrap(err, "prepare solver")
 			}
 
-			if cmd.noVendor && cmd.dryRun {
-				return errors.New("Gopkg.lock was not up to date")
-			}
-
 			solution, err := solver.Solve(context.TODO())
 			if err != nil {
 				return handleAllTheFailuresOfTheWorld(err)
@@ -306,23 +313,22 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
 			// The user said not to touch vendor/, so definitely nothing to do.
 			return nil
 		}
-
 	}
 
-	sw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"))
+	dw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"))
 	if err != nil {
 		return err
 	}
 
 	if cmd.dryRun {
-		return sw.PrintPreparedActions(ctx.Out, ctx.Verbose)
+		return dw.PrintPreparedActions(ctx.Out, ctx.Verbose)
 	}
 
 	var logger *log.Logger
 	if ctx.Verbose {
 		logger = ctx.Err
 	}
-	return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
+	return errors.WithMessage(dw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
 }
 
 func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
@@ -333,9 +339,10 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
 	if p.Lock == nil {
 		return errors.Errorf("no %s exists from which to populate vendor/", dep.LockName)
 	}
+
 	// Pass the same lock as old and new so that the writer will observe no
 	// difference and choose not to write it out.
-	sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions)
+	sw, err := dep.NewSafeWriter(nil, p.Lock, p.ChangedLock, dep.VendorAlways, p.Manifest.PruneOptions)
 	if err != nil {
 		return err
 	}
@@ -383,19 +390,19 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
 		return handleAllTheFailuresOfTheWorld(err)
 	}
 
-	sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior(), p.Manifest.PruneOptions)
+	dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"))
 	if err != nil {
 		return err
 	}
 	if cmd.dryRun {
-		return sw.PrintPreparedActions(ctx.Out, ctx.Verbose)
+		return dw.PrintPreparedActions(ctx.Out, ctx.Verbose)
 	}
 
 	var logger *log.Logger
 	if ctx.Verbose {
 		logger = ctx.Err
 	}
-	return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
+	return errors.Wrap(dw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
 }
 
 func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error {
@@ -417,16 +424,6 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
 
 	rm, _ := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages())
 
-	// TODO(sdboyer) re-enable this once we ToReachMap() intelligently filters out normally-excluded (_*, .*), dirs from errmap
-	//rm, errmap := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages())
-	// Having some problematic internal packages isn't cause for termination,
-	// but the user needs to be warned.
-	//for fail, err := range errmap {
-	//if _, is := err.Err.(*build.NoGoError); !is {
-	//ctx.Err.Printf("Warning: %s, %s", fail, err)
-	//}
-	//}
-
 	// Compile unique sets of 1) all external packages imported or required, and
 	// 2) the project roots under which they fall.
 	exmap := make(map[string]bool)
@@ -673,20 +670,20 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
 	}
 	sort.Strings(reqlist)
 
-	sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), dep.VendorOnChanged, p.Manifest.PruneOptions)
+	dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"))
 	if err != nil {
 		return err
 	}
 
 	if cmd.dryRun {
-		return sw.PrintPreparedActions(ctx.Out, ctx.Verbose)
+		return dw.PrintPreparedActions(ctx.Out, ctx.Verbose)
 	}
 
 	var logger *log.Logger
 	if ctx.Verbose {
 		logger = ctx.Err
 	}
-	if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil {
+	if err := errors.Wrap(dw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil {
 		return err
 	}
 

cmd/dep/ensure.go

@@ -102,12 +102,12 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 		ctx.Out.Println("Getting direct dependencies...")
 	}
 
-	ptree, directDeps, err := p.GetDirectDependencyNames(sm)
+	directDeps, err := p.GetDirectDependencyNames(sm)
 	if err != nil {
 		return errors.Wrap(err, "init failed: unable to determine direct dependencies")
 	}
 	if ctx.Verbose {
-		ctx.Out.Printf("Checked %d directories for packages.\nFound %d direct dependencies.\n", len(ptree.Packages), len(directDeps))
+		ctx.Out.Printf("Checked %d directories for packages.\nFound %d direct dependencies.\n", len(p.RootPackageTree.Packages), len(directDeps))
 	}
 
 	// Initialize with imported data, then fill in the gaps using the GOPATH
@@ -133,7 +133,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 
 	params := gps.SolveParameters{
 		RootDir:         root,
-		RootPackageTree: ptree,
+		RootPackageTree: p.RootPackageTree,
 		Manifest:        p.Manifest,
 		Lock:            p.Lock,
 		ProjectAnalyzer: rootAnalyzer,

cmd/dep/init.go

@@ -329,13 +329,13 @@ type dotOutput struct {
 func (out *dotOutput) BasicHeader() error {
 	out.g = new(graphviz).New()
 
-	ptree, err := out.p.ParseRootPackageTree()
+	ptree := out.p.RootPackageTree
 	// TODO(sdboyer) should be true, true, false, out.p.Manifest.IgnoredPackages()
 	prm, _ := ptree.ToReachMap(true, false, false, nil)
 
 	out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath))
 
-	return err
+	return nil
 }
 
 func (out *dotOutput) BasicFooter() error {
@@ -649,10 +649,7 @@ func (os OldStatus) marshalJSON() *rawOldStatus {
 func (cmd *statusCommand) runOld(ctx *dep.Ctx, out oldOutputter, p *dep.Project, sm gps.SourceManager) error {
 	// While the network churns on ListVersions() requests, statically analyze
 	// code from the current project.
-	ptree, err := p.ParseRootPackageTree()
-	if err != nil {
-		return err
-	}
+	ptree := p.RootPackageTree
 
 	// Set up a solver in order to check the InputHash.
 	params := gps.SolveParameters{
@@ -888,10 +885,7 @@ type MissingStatus struct {
 func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (hasMissingPkgs bool, errCount int, err error) {
 	// While the network churns on ListVersions() requests, statically analyze
 	// code from the current project.
-	ptree, err := p.ParseRootPackageTree()
-	if err != nil {
-		return false, 0, err
-	}
+	ptree := p.RootPackageTree
 
 	// Set up a solver in order to check the InputHash.
 	params := gps.SolveParameters{
@@ -928,7 +922,7 @@ func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Proje
 		return slp[i].Ident().Less(slp[j].Ident())
 	})
 
-	lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree)
+	lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree)
 	if lsat.Passed() {
 		// If these are equal, we're guaranteed that the lock is a transitively
 		// complete picture of all deps. That eliminates the need for at least
@@ -1305,7 +1299,7 @@ func collectConstraints(ctx *dep.Ctx, p *dep.Project, sm gps.SourceManager) (con
 
 	// Collect the complete set of direct project dependencies, incorporating
 	// requireds and ignores appropriately.
-	_, directDeps, err := p.GetDirectDependencyNames(sm)
+	directDeps, err := p.GetDirectDependencyNames(sm)
 	if err != nil {
 		// Return empty collection, not nil, if we fail here.
 		return constraintCollection, []error{errors.Wrap(err, "failed to get direct dependencies")}

cmd/dep/status.go

@@ -9,9 +9,13 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"time"
 
 	"github.com/golang/dep/gps"
+	"github.com/golang/dep/gps/paths"
+	"github.com/golang/dep/gps/pkgtree"
+	"github.com/golang/dep/gps/verify"
 	"github.com/golang/dep/internal/fs"
 	"github.com/pkg/errors"
 )
@@ -188,9 +192,55 @@ func (c *Ctx) LoadProject() (*Project, error) {
 		return nil, errors.Wrapf(err, "error while parsing %s", lp)
 	}
 
+	// Parse in the root package tree.
+	ptree, err := p.parseRootPackageTree()
+	if err != nil {
+		return nil, err
+	}
+
+	// If there's a current Lock, apply the input and pruneopt changes that we
+	// can know without solving.
+	if p.Lock != nil {
+		p.ChangedLock = p.Lock.dup()
+		p.ChangedLock.SolveMeta.InputImports = externalImportList(ptree, p.Manifest)
+
+		for k, lp := range p.ChangedLock.Projects() {
+			vp := lp.(verify.VerifiableProject)
+			vp.PruneOpts = p.Manifest.PruneOptions.PruneOptionsFor(lp.Ident().ProjectRoot)
+			p.ChangedLock.P[k] = vp
+		}
+	}
+
 	return p, nil
 }
 
+func externalImportList(rpt pkgtree.PackageTree, m gps.RootManifest) []string {
+	rm, _ := rpt.ToReachMap(true, true, false, m.IgnoredPackages())
+	reach := rm.FlattenFn(paths.IsStandardImportPath)
+	req := m.RequiredPackages()
+
+	// If there are any requires, slide them into the reach list, as well.
+	if len(req) > 0 {
+		// Make a map of imports that are both in the import path list and the
+		// required list to avoid duplication.
+		skip := make(map[string]bool, len(req))
+		for _, r := range reach {
+			if req[r] {
+				skip[r] = true
+			}
+		}
+
+		for r := range req {
+			if !skip[r] {
+				reach = append(reach, r)
+			}
+		}
+	}
+
+	sort.Strings(reach)
+	return reach
+}
+
 // DetectProjectGOPATH attempt to find the GOPATH containing the project.
 //
 //  If p.AbsRoot is not a symlink and is within a GOPATH, the GOPATH containing p.AbsRoot is returned.

context.go

@@ -461,7 +461,9 @@ func VerifyDepTree(osDirname string, wantDigests map[string]VersionedDigest) (ma
 		if expectedSum, ok := wantDigests[slashPathname]; ok {
 			ls := EmptyDigestInLock
 			if expectedSum.HashVersion != HashVersion {
-				ls = HashVersionMismatch
+				if !expectedSum.IsEmpty() {
+					ls = HashVersionMismatch
+				}
 			} else if len(expectedSum.Digest) > 0 {
 				projectSum, err := DigestFromDirectory(osPathname)
 				if err != nil {