Merge branch 'master' into unit_tests

Author: tariq1890

.codeclimate.yml

@@ -22,7 +22,9 @@ checks:
     enabled: false
 plugins:
   gofmt:
-    enabled: true
+    # Codeclimate go fmt does not agree with tip go fmt; consider re-enabling
+    # CC when the advice matches up with tip again.
+    enabled: false
   govet:
     enabled: true
   golint:

.travis.yml

@@ -1,44 +1,61 @@
 language: go
 sudo: false
+dist: xenial
 notifications:
   email: false
 jobs:
   include:
     - stage: test
       go_import_path: github.com/golang/dep
       install:
+        - ssh-keyscan -t $TRAVIS_SSH_KEY_TYPES -H bitbucket.org >> ~/.ssh/known_hosts
         - make get-deps
         - npm install -g codeclimate-test-reporter
       env:
         - DEPTESTBYPASS501=1
+        - TZ=UTC
       os: linux
-      go: 1.10.x
+      go: 1.11.x
       script:
         - make validate test
         - ./hack/coverage.bash
       after_success:
         - codeclimate-test-reporter < coverage.txt
     # YAML alias, for settings shared across the simpler builds
     - &simple-test
-      go: 1.9.x
+      go: 1.10.x
       stage: test
       go_import_path: github.com/golang/dep
-      install: skip
+      install:
+        - ssh-keyscan -t $TRAVIS_SSH_KEY_TYPES -H bitbucket.org >> ~/.ssh/known_hosts
       env:
         - DEPTESTBYPASS501=1
+        - TZ=UTC
       script: make test
+    - <<: *simple-test
+      go: 1.9.x
     - <<: *simple-test
       go: tip
+      install:
+        - ssh-keyscan -t $TRAVIS_SSH_KEY_TYPES -H bitbucket.org >> ~/.ssh/known_hosts
+        - mkdir -p /home/travis/var/cache
+      env:
+        - GOCACHE=/home/travis/var/cache
+        - DEPTESTBYPASS501=1
+        - TZ=UTC
+
     - <<: *simple-test
       os: osx
-      go: 1.10.x
+      go: 1.11.x
       install:
         # brew takes horribly long to update itself despite the above caching
         # attempt; only bzr install if it's not on the $PATH
+        - ssh-keyscan -t $TRAVIS_SSH_KEY_TYPES -H bitbucket.org >> ~/.ssh/known_hosts
         - test $(which bzr) || brew install bzr
       env:
         - HOMEBREW_NO_AUTO_UPDATE=1
         - DEPTESTBYPASS501=1
+        - TZ=UTC
       script:
         # OSX as of El Capitan sets an exit trap that interacts poorly with how
         # travis seems to spawn these shells; if set -e is set, then it can cause
@@ -47,13 +64,14 @@ jobs:
         # Related: https://superuser.com/questions/1044130/why-am-i-having-how-can-i-fix-this-error-shell-session-update-command-not-f
         - trap EXIT
         - make test
-    - go: 1.10.x
+    - go: 1.11.x
       # Run on OS X so that we get a CGO-enabled binary for this OS; see
       # https://github.com/golang/dep/issues/1838 for more details.
       os: osx
       stage: deploy
       go_import_path: github.com/golang/dep
-      install: skip
+      install:
+        - ssh-keyscan -t $TRAVIS_SSH_KEY_TYPES -H bitbucket.org >> ~/.ssh/known_hosts
       script: skip
       before_deploy:
         - ./hack/build-all.bash
@@ -82,6 +100,8 @@ jobs:
             - release/dep-linux-ppc64.sha256
             - release/dep-linux-ppc64le
             - release/dep-linux-ppc64le.sha256
+            - release/dep-linux-s390x
+            - release/dep-linux-s390x.sha256
           skip_cleanup: true
           on:
             repo: golang/dep

CHANGELOG.md

@@ -1,5 +1,9 @@
 # v0.5.1
 
+IMPROVEMENTS:
+
+* The `noverify` field in `Gopkg.toml` allows for the preservation of excess files under `vendor`. ([#2002](https://github.com/golang/dep/issue/2002))
+
 BUG FIXES:
 
 * Correctly handle certain cases where `dep ensure` removed projects from Gopkg.lock. ([#1945](https://github.com/golang/dep/issue/1945)).

Gopkg.lock

@@ -141,62 +141,83 @@ func (cmd *checkCommand) Run(ctx *dep.Ctx, args []string) error {
 			noverify[skip] = true
 		}
 
-		var vendorfail bool
+		var vendorfail, hasnoverify bool
 		// One full pass through, to see if we need to print the header, and to
 		// create an array of names to sort for deterministic output.
 		var ordered []string
 		for path, status := range statuses {
 			ordered = append(ordered, path)
 
 			switch status {
-			case verify.DigestMismatchInLock, verify.HashVersionMismatch, verify.EmptyDigestInLock:
-				// NoVerify applies only to these three cases.
+			case verify.DigestMismatchInLock, verify.HashVersionMismatch, verify.EmptyDigestInLock, verify.NotInLock:
 				if noverify[path] {
+					hasnoverify = true
 					continue
 				}
 				fallthrough
-			case verify.NotInTree, verify.NotInLock:
+			case verify.NotInTree:
+				// NoVerify cannot be used to make dep check ignore the absence
+				// of a project entirely.
+				if noverify[path] {
+					delete(noverify, path)
+				}
+
 				fail = true
 				if !vendorfail {
 					vendorfail = true
-					logger.Println("# vendor is out of sync:")
 				}
-
 			}
 		}
 		sort.Strings(ordered)
 
+		var vfbuf, novbuf bytes.Buffer
+		var bufptr *bytes.Buffer
+
+		fmt.Fprintf(&vfbuf, "# vendor is out of sync:\n")
+		fmt.Fprintf(&novbuf, "# out of sync, but ignored, due to noverify in Gopkg.toml:\n")
+
 		for _, pr := range ordered {
-			var nvSuffix string
 			if noverify[pr] {
-				nvSuffix = "  (CHECK IGNORED: marked noverify in Gopkg.toml)"
+				bufptr = &novbuf
+			} else {
+				bufptr = &vfbuf
 			}
 
 			status := statuses[pr]
 			switch status {
 			case verify.NotInTree:
-				logger.Printf("%s: missing from vendor\n", pr)
+				fmt.Fprintf(bufptr, "%s: missing from vendor\n", pr)
 			case verify.NotInLock:
 				fi, err := os.Stat(filepath.Join(p.AbsRoot, "vendor", pr))
 				if err != nil {
 					return errors.Wrap(err, "could not stat file that VerifyVendor claimed existed")
 				}
 				if fi.IsDir() {
-					logger.Printf("%s: unused project\n", pr)
+					fmt.Fprintf(bufptr, "%s: unused project\n", pr)
 				} else {
-					logger.Printf("%s: orphaned file\n", pr)
+					fmt.Fprintf(bufptr, "%s: orphaned file\n", pr)
 				}
 			case verify.DigestMismatchInLock:
-				logger.Printf("%s: hash of vendored tree not equal to digest in Gopkg.lock%s\n", pr, nvSuffix)
+				fmt.Fprintf(bufptr, "%s: hash of vendored tree not equal to digest in Gopkg.lock\n", pr)
 			case verify.EmptyDigestInLock:
-				logger.Printf("%s: no digest in Gopkg.lock to compare against hash of vendored tree%s\n", pr, nvSuffix)
+				fmt.Fprintf(bufptr, "%s: no digest in Gopkg.lock to compare against hash of vendored tree\n", pr)
 			case verify.HashVersionMismatch:
 				// This will double-print if the hash version is zero, but
 				// that's a rare case that really only occurs before the first
 				// run with a version of dep >=0.5.0, so it's fine.
-				logger.Printf("%s: hash algorithm mismatch, want version %v%s\n", pr, verify.HashVersion, nvSuffix)
+				fmt.Fprintf(bufptr, "%s: hash algorithm mismatch, want version %v\n", pr, verify.HashVersion)
 			}
 		}
+
+		if vendorfail {
+			logger.Print(vfbuf.String())
+			if hasnoverify {
+				logger.Println()
+			}
+		}
+		if hasnoverify {
+			logger.Print(novbuf.String())
+		}
 	}
 
 	if fail {

cmd/dep/check.go

@@ -35,10 +35,6 @@ func TestMain(m *testing.M) {
 		os.Setenv("CCACHE_DIR", filepath.Join(home, ".ccache"))
 	}
 	os.Setenv("HOME", "/test-dep-home-does-not-exist")
-	if os.Getenv("GOCACHE") == "" {
-		os.Setenv("GOCACHE", "off") // because $HOME is gone
-	}
-
 	r := m.Run()
 
 	os.Remove("testdep" + test.ExeSuffix)

cmd/dep/dep_test.go

@@ -74,7 +74,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 	}
 
 	var root string
-	if len(args) <= 0 {
+	if len(args) == 0 {
 		root = ctx.WorkingDir
 	} else {
 		root = args[0]

cmd/dep/init.go

@@ -340,7 +340,7 @@ func (out *dotOutput) BasicHeader() error {
 
 func (out *dotOutput) BasicFooter() error {
 	gvo := out.g.output("")
-	_, err := fmt.Fprintf(out.w, gvo.String())
+	_, err := fmt.Fprint(out.w, gvo.String())
 	return err
 }
 

cmd/dep/status.go

@@ -7,6 +7,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"path/filepath"
@@ -16,8 +17,6 @@ import (
 	"text/tabwriter"
 	"text/template"
 
-	"io"
-
 	"github.com/golang/dep"
 	"github.com/golang/dep/gps"
 	"github.com/golang/dep/internal/test"
@@ -28,7 +27,7 @@ func TestStatusFormatVersion(t *testing.T) {
 	t.Parallel()
 
 	tests := map[gps.Version]string{
-		nil: "",
+		nil:                            "",
 		gps.NewBranch("master"):        "branch master",
 		gps.NewVersion("1.0.0"):        "1.0.0",
 		gps.Revision("flooboofoobooo"): "flooboo",

cmd/dep/status_test.go

@@ -1 +1,2 @@
-github.com/sdboyer/deptest: hash of vendored tree not equal to digest in Gopkg.lock  (CHECK IGNORED: marked noverify in Gopkg.toml)
+# out of sync, but ignored, due to noverify in Gopkg.toml:
+github.com/sdboyer/deptest: hash of vendored tree not equal to digest in Gopkg.lock

cmd/dep/testdata/harness_tests/check/noverify/hash_mismatch/stdout.txt

@@ -1 +1,2 @@
-github.com/sdboyer/deptest: hash algorithm mismatch, want version 1  (CHECK IGNORED: marked noverify in Gopkg.toml)
+# out of sync, but ignored, due to noverify in Gopkg.toml:
+github.com/sdboyer/deptest: hash algorithm mismatch, want version 1

cmd/dep/testdata/harness_tests/check/noverify/hash_version_mismatch/stdout.txt

@@ -2,4 +2,4 @@ This is a hack - it's effectively just verifying that the Gopkg.lock doesn't
 change for projects with noverify set, which (under the current logic) is an
 indicator that vendor wasn't updated.
 
-Of course, that vendor -> lock relatinoship isn't guaranteed to hold...
+Of course, that vendor -> lock relationship isn't guaranteed to hold...

cmd/dep/testdata/harness_tests/ensure/noverify/hash_mismatch/README

@@ -0,0 +1 @@
+noverify = ["foo", "orphdir"]

cmd/dep/testdata/harness_tests/ensure/noverify/vendororphans/final/Gopkg.lock

@@ -0,0 +1 @@
+noverify = ["foo", "orphdir"]

cmd/dep/testdata/harness_tests/ensure/noverify/vendororphans/final/Gopkg.toml

@@ -0,0 +1,12 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "github.com/sdboyer/deptest"
+)
+
+func main() {
+}

cmd/dep/testdata/harness_tests/ensure/noverify/vendororphans/initial/Gopkg.lock

@@ -0,0 +1,3 @@
+# out of sync, but ignored, due to noverify in Gopkg.toml:
+foo: orphaned file
+orphdir: unused project

cmd/dep/testdata/harness_tests/ensure/noverify/vendororphans/initial/Gopkg.toml

@@ -0,0 +1,9 @@
+{
+  "commands": [
+    ["ensure"],
+    ["check"]
+  ],
+  "vendor-final": [
+    "github.com/sdboyer/deptest"
+  ]
+}

cmd/dep/testdata/harness_tests/ensure/noverify/vendororphans/initial/main.go

@@ -231,18 +231,20 @@ It is usually safe to set `non-go = true`, as well. However, as dep only has a c
 
 ## `noverify`
 
-The `noverify` field is a list of [project roots](glossary.md#project-root) to exclude from [vendor verification](glossary.md#vendor-verification).
+The `noverify` field is a list of paths, typically [project roots](glossary.md#project-root), to exclude from [vendor verification](glossary.md#vendor-verification).
 
 Dep uses per-project hash digests, computed after pruning and recorded in [Gopkg.lock](Gopkg.lock.md#digest), to determine if the contents of `vendor/` are as expected. If the recorded digest and the hash of the corresponding tree in `vendor/` differ, that project is considered to be out of sync:
 
 * `dep ensure` will regenerate it
 * `dep check` will complain of a hash mismatch and exit 1
 
-It is strongly preferable for almost all workflows that you leave `vendor/` unmodified, in whatever state dep puts it in. However, this isn't always an option. If you have no choice but to modify `vendor/` for a particular project, then add the project root for that project to `noverify`. This will have the following effects:
+It is strongly recommended that you leave `vendor/` unmodified, in whatever state dep puts it in. However, this isn't always feasible. If you have no choice but to modify `vendor/` for a particular project, then add the project root for that project to `noverify`. This will have the following effects:
 
 * `dep ensure` will ignore hash mismatches for the project, and only regenerate it in `vendor/` if absolutely necessary (prune options change, package list changes, version changes)
 * `dep check` will continue to report hash mismatches (albeit with an annotation about `noverify`) for the project, but will no longer exit 1. 
 
+`noverify` can also be used to preserve certain excess paths that would otherwise be removed; for example, adding `WORKSPACE` to the `noverify` list would allow you to preserve `vendor/WORKSPACE`, which can help with some Bazel-based workflows.
+
 ## Scope
 
 `dep` evaluates

cmd/dep/testdata/harness_tests/ensure/noverify/vendororphans/initial/vendor/foo

@@ -13,7 +13,7 @@ Briefly, the four states are:
 1.  The [current project's](glossary.md#current-project) source code.
 2.  A [manifest](glossary.md#manifest) - a file describing the current project's dependency requirements. In dep, this is the [`Gopkg.toml`](Gopkg.toml.md) file.
 3.  A [lock](glossary.md#lock) - a file containing a transitively-complete, reproducible description of the dependency graph. In dep, this is the [`Gopkg.lock`](Gopkg.lock.md) file.
-4.  The source code of the dependences themselves. In dep's current design, this is the `vendor/` directory.
+4.  The source code of the dependencies themselves. In dep's current design, this is the `vendor/` directory.
 
 We can visually represent these four states as follows:
 
@@ -156,7 +156,7 @@ Ordinarily, when the solver encounters a project name for which there's an entry
 
 "Skips pulling the version from the lock" would imply that `dep ensure -update github.com/foo/bar` is equivalent to removing the `[[project]]` stanza for `github.com/foo/bar` from your `Gopkg.lock`, then running `dep ensure`. And indeed it is - however, that approach is not recommended, and subtle changes may be introduced in the future that complicate the equivalency.
 
-If `-update` is passed with no arguments, then `ChangeAll` is set to `true`, resulting in the solver ignoring `Gopkg.lock` for all newly-encountered project names. This is equivalent to explicitly passing all of your dependences as arguments to `dep ensure -update`, as well as `rm Gopkg.lock && dep ensure`. Again, however, neither of these approaches are recommended, and future changes may introduce subtle differences.
+If `-update` is passed with no arguments, then `ChangeAll` is set to `true`, resulting in the solver ignoring `Gopkg.lock` for all newly-encountered project names. This is equivalent to explicitly passing all of your dependencies as arguments to `dep ensure -update`, as well as `rm Gopkg.lock && dep ensure`. Again, however, neither of these approaches are recommended, and future changes may introduce subtle differences.
 
 When a version hint from `Gopkg.lock` is not placed at the head of the version queue, it means that dep will explore the set of possible versions for a particular dependency. This exploration is performed according to a [fixed sort order](https://godoc.org/github.com/golang/dep/gps#SortForUpgrade), where newer versions are tried first, resulting in an update.