script/genproto.sh: Refactor to be explicit about versions.

Refactoring script/genproto.sh around state-of-the-art techniques of
managing tooling in go:
  - https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module
  - uses https://github.com/myitcv/gobin instead of customly created gopath.proto dir
  - caches tools between executions
  - guaratees hermetics runs (it was not guaranteed for protoc_grpc_gateway that used latest)

The change is no-op for the generated code.

The commit reveals a few 'worring things':
  1  We depend on : github.com/grpc-ecosystem/grpc-gateway/@v/v1.4.1/protoc-gen-grpc-gateway
  2. And also     : github.com/grpc-ecosystem/grpc-gateway/@v/v1.15.0/protoc-gen-swagger/protoc-gen-swagger
  3. And on extremely old: github.com/gogo/[email protected] protoc-gen-gofast that is out of sync with the library linked to binaries: github.com/gogo/[email protected]

Author: ptabor

.gitignore

@@ -19,5 +19,5 @@ hack/tls-setup/certs
 /vendor
 /tests/e2e/default.proxy
 *.tmp
-
 *.bak
+.gobincache/

scripts/genproto.sh

@@ -10,65 +10,42 @@ if ! [[ "$0" =~ scripts/genproto.sh ]]; then
 	exit 255
 fi
 
+source ./scripts/test_lib.sh
+
 if [[ $(protoc --version | cut -f2 -d' ') != "3.12.3" ]]; then
 	echo "could not find protoc 3.12.3, is it installed + in PATH?"
 	exit 255
 fi
 
-# directories containing protos to be built
-DIRS="./wal/walpb ./api/etcdserverpb ./etcdserver/api/snap/snappb ./raft/raftpb ./api/mvccpb ./lease/leasepb ./api/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb ./api/membershippb"
+export GOMOD="./tools/mod"
+
+run env GO111MODULE=off go get -u github.com/myitcv/gobin
 
-# disable go mod - this is for the go get/install invocations
-export GO111MODULE=off
+GOFAST_BIN=$(tool_get_bin github.com/gogo/protobuf/protoc-gen-gofast)
+GRPC_GATEWAY_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)
+SWAGGER_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/[email protected])
+GOGOPROTO_ROOT="$(tool_pkg_dir github.com/gogo/protobuf/proto)/.."
+GRPC_GATEWAY_ROOT="$(tool_pkg_dir github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)/.."
 
-# exact version of packages to build
-GOGO_PROTO_SHA="1adfc126b41513cc696b209667c8656ea7aac67c"
-GRPC_GATEWAY_SHA="92583770e3f01b09a0d3e9bdf64321d8bebd48f2"
-SCHWAG_SHA="b7d0fc9aadaaae3d61aaadfc12e4a2f945514912"
+echo
+echo "Resolved binary and packages versions:"
+echo "  - protoc-gen-gofast:       ${GOFAST_BIN}"
+echo "  - protoc-gen-grpc-gateway: ${GRPC_GATEWAY_BIN}"
+echo "  - swagger:                 ${SWAGGER_BIN}"
+echo "  - gogoproto-root:          ${GOGOPROTO_ROOT}"
+echo "  - grpc-gateway-root:       ${GRPC_GATEWAY_ROOT}"
 
-# set up self-contained GOPATH for building
-export GOPATH=${PWD}/gopath.proto
-export GOBIN=${PWD}/bin
-export PATH="${GOBIN}:${PATH}"
+# directories containing protos to be built
+DIRS="./wal/walpb ./api/etcdserverpb ./etcdserver/api/snap/snappb ./raft/raftpb ./api/mvccpb ./lease/leasepb ./api/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb ./api/membershippb"
 
-ETCD_IO_ROOT="${GOPATH}/src/go.etcd.io"
-ETCD_ROOT="${ETCD_IO_ROOT}/etcd"
-GOGOPROTO_ROOT="${GOPATH}/src/github.com/gogo/protobuf"
-SCHWAG_ROOT="${GOPATH}/src/github.com/hexfusion/schwag"
 GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf"
-GRPC_GATEWAY_ROOT="${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway"
-
-function cleanup {
-  # Remove the whole fake GOPATH which can really confuse go mod.
-  rm -rf "${PWD}/gopath.proto"
-}
-
-cleanup
-trap cleanup EXIT
-
-mkdir -p "${ETCD_IO_ROOT}"
-ln -s "${PWD}" "${ETCD_ROOT}"
-
-# Ensure we have the right version of protoc-gen-gogo by building it every time.
-# TODO(jonboulle): vendor this instead of `go get`ting it.
-go get -u github.com/gogo/protobuf/{proto,protoc-gen-gogo,gogoproto}
-go get -u golang.org/x/tools/cmd/goimports
-pushd "${GOGOPROTO_ROOT}"
-	git reset --hard "${GOGO_PROTO_SHA}"
-	make install
-popd
-
-# generate gateway code
-go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
-pushd "${GRPC_GATEWAY_ROOT}"
-	git reset --hard "${GRPC_GATEWAY_SHA}"
-	go install ./protoc-gen-grpc-gateway
-popd
+
+log_callout -e "\nRunning gofast proto generation..."
 
 for dir in ${DIRS}; do
-	pushd "${dir}"
-		protoc --gofast_out=plugins=grpc,import_prefix=go.etcd.io/:. -I=".:${GOGOPROTO_PATH}:${ETCD_IO_ROOT}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" ./*.proto
+	run pushd "${dir}"
+		run protoc --gofast_out=plugins=grpc,import_prefix=go.etcd.io/:. -I=".:${GOGOPROTO_PATH}:${ETCD_ROOT_DIR}/..:${GRPC_GATEWAY_ROOT}/third_party/googleapis" \
+		  --plugin="${GOFAST_BIN}" ./*.proto
 		# shellcheck disable=SC1117
 		sed -i.bak -E 's/go\.etcd\.io\/(gogoproto|github\.com|golang\.org|google\.golang\.org)/\1/g' ./*.pb.go
 		# shellcheck disable=SC1117
@@ -86,30 +63,30 @@ for dir in ${DIRS}; do
 		# shellcheck disable=SC1117
 		sed -i.bak -E "s|go.etcd.io/etcd/v3/api|go.etcd.io/etcd/api/v3|g" ./*.pb.go
 		rm -f ./*.bak
-		gofmt -s -w ./*.pb.go
-		goimports -w ./*.pb.go
-	popd
+		run gofmt -s -w ./*.pb.go
+		run goimports -w ./*.pb.go
+	run popd
 done
 
+log_callout -e "\nRunning swagger & grpc_gateway proto generation..."
+
 # remove old swagger files so it's obvious whether the files fail to generate
 rm -rf Documentation/dev-guide/apispec/swagger/*json
 for pb in api/etcdserverpb/rpc etcdserver/api/v3lock/v3lockpb/v3lock etcdserver/api/v3election/v3electionpb/v3election; do
 	protobase="${pb}"
-	protoc -I. \
+	run protoc -I. \
 	    -I"${GRPC_GATEWAY_ROOT}"/third_party/googleapis \
 	    -I"${GOGOPROTO_PATH}" \
-	    -I"${ETCD_IO_ROOT}" \
+	    -I"${ETCD_ROOT_DIR}/.." \
 	    --grpc-gateway_out=logtostderr=true:. \
 	    --swagger_out=logtostderr=true:./Documentation/dev-guide/apispec/swagger/. \
+	    --plugin="${SWAGGER_BIN}" --plugin="${GRPC_GATEWAY_BIN}" \
 	    ${protobase}.proto
 	# hack to move gw files around so client won't include them
 	pkgpath=$(dirname "${protobase}")
 	pkg=$(basename "${pkgpath}")
 	gwfile="${protobase}.pb.gw.go"
 
-	echo ">>> $gwfile"
-	head -n 30 "$gwfile"
-
 	sed -i.bak -E "s/package $pkg/package gw/g" ${gwfile}
 	# shellcheck disable=SC1117
 	sed -i.bak -E "s/protoReq /&$pkg\./g" ${gwfile}
@@ -125,52 +102,36 @@ for pb in api/etcdserverpb/rpc etcdserver/api/v3lock/v3lockpb/v3lock etcdserver/
 	# shellcheck disable=SC1117
 	sed -i.bak -E "s|go.etcd.io/etcd/v3/api|go.etcd.io/etcd/api/v3|g" ${gwfile} 
 	mkdir -p  "${pkgpath}"/gw/
-	go fmt ${gwfile}
+	run go fmt ${gwfile}
 	mv ${gwfile} "${pkgpath}/gw/"
 	rm -f ./${protobase}*.bak
 	swaggerName=$(basename ${protobase})
-	mv	Documentation/dev-guide/apispec/swagger/${protobase}.swagger.json \
+	run mv	Documentation/dev-guide/apispec/swagger/${protobase}.swagger.json \
 		Documentation/dev-guide/apispec/swagger/"${swaggerName}".swagger.json
 done
-rm -rf Documentation/dev-guide/apispec/swagger/etcdserver/
-
-# append security to swagger spec
-go get -u "github.com/hexfusion/schwag"
-pushd "${SCHWAG_ROOT}"
-	git reset --hard "${SCHWAG_SHA}"
-	go install .
-popd
-schwag -input=Documentation/dev-guide/apispec/swagger/rpc.swagger.json
-
-# install protodoc
-# go get -v -u go.etcd.io/protodoc
-#
-# run './scripts/genproto.sh --skip-protodoc'
-# to skip protodoc generation
-#
+
+log_callout -e "\nRunning swagger ..."
+run_go_tool github.com/hexfusion/schwag -input=Documentation/dev-guide/apispec/swagger/rpc.swagger.json
+
 if [ "$1" != "--skip-protodoc" ]; then
-	echo "protodoc is auto-generating grpc API reference documentation..."
-	go get -v -u go.etcd.io/protodoc
-	SHA_PROTODOC="484ab544e116302a9a6021cc7c427d334132e94a"
-	PROTODOC_PATH="${GOPATH}/src/go.etcd.io/protodoc"
-	pushd "${PROTODOC_PATH}"
-		git reset --hard "${SHA_PROTODOC}"
-		go install
-		echo "protodoc is updated"
-	popd
-
-	protodoc --directories="api/etcdserverpb=service_message,api/mvccpb=service_message,lease/leasepb=service_message,api/authpb=service_message" \
+	log_callout "protodoc is auto-generating grpc API reference documentation..."
+
+  run rm -rf Documentation/dev-guide/api_reference_v3.md
+	run_go_tool go.etcd.io/protodoc --directories="api/etcdserverpb=service_message,api/mvccpb=service_message,lease/leasepb=service_message,api/authpb=service_message" \
 		--title="etcd API Reference" \
 		--output="Documentation/dev-guide/api_reference_v3.md" \
 		--message-only-from-this-file="api/etcdserverpb/rpc.proto" \
-		--disclaimer="This is a generated documentation. Please read the proto files for more."
+		--disclaimer="This is a generated documentation. Please read the proto files for more." || exit 2
 
-	protodoc --directories="etcdserver/api/v3lock/v3lockpb=service_message,etcdserver/api/v3election/v3electionpb=service_message,api/mvccpb=service_message" \
+  run rm -rf Documentation/dev-guide/api_concurrency_reference_v3.md
+	run_go_tool go.etcd.io/protodoc --directories="etcdserver/api/v3lock/v3lockpb=service_message,etcdserver/api/v3election/v3electionpb=service_message,api/mvccpb=service_message" \
 		--title="etcd concurrency API Reference" \
 		--output="Documentation/dev-guide/api_concurrency_reference_v3.md" \
-		--disclaimer="This is a generated documentation. Please read the proto files for more."
+		--disclaimer="This is a generated documentation. Please read the proto files for more." || exit 2
 
-	echo "protodoc is finished..."
+	log_success "protodoc is finished."
 else
-	echo "skipping grpc API reference document auto-generation..."
+	log_warning "skipping grpc API reference document auto-generation..."
 fi
+
+log_success -e "\n./genproto SUCCESS"

scripts/test_lib.sh

@@ -191,6 +191,9 @@ function go_test {
 # tool_exists [tool] [instruction]
 # Checks whether given [tool] is installed. In case of failure,
 # prints a warning with installation [instruction] and returns !=0 code.
+#
+# WARNING: This depend on "any" version of the 'binary' that might be tricky
+# from hermetic build perspective. For go binaries prefer 'tool_go_run'
 function tool_exists {
   local tool="${1}"
   local instruction="${2}"
@@ -200,3 +203,31 @@ function tool_exists {
   fi
 }
 
+# tool_get_bin [tool] - returns absolute path to a tool binary (or returns error)
+function tool_get_bin {
+  tool_exists "gobin" "GO111MODULE=off go get -u github.com/myitcv/gobin" || return 2
+
+  local tool="$1"
+  if [[ "$tool" == *"@"* ]]; then
+    run gobin -p "${tool}" || return 2
+  else
+    run_for_module ./tools/mod run gobin -p -m --mod=readonly "${tool}" || return 2
+  fi
+}
+
+# tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg.
+# The pkg versions must be defined in ./tools/mod directory.
+function tool_pkg_dir {
+  run_for_module ./tools/mod run go list -f '{{.Dir}}' "${1}"
+}
+
+# tool_get_bin [tool]
+function run_go_tool {
+  local cmdbin
+  if ! cmdbin=$(tool_get_bin "${1}"); then
+    return 2
+  fi
+  shift 1
+  run "${cmdbin}" "$@" || return 2
+}
+

tools/mod/go.mod

@@ -0,0 +1,15 @@
+module go.etcd.io/etcd/tools/v3
+
+go 1.15
+
+require (
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-openapi/loads v0.19.5 // indirect
+	github.com/go-openapi/spec v0.19.9 // indirect
+	github.com/gogo/protobuf v1.0.0
+	github.com/grpc-ecosystem/grpc-gateway v1.4.1
+	github.com/hexfusion/schwag v0.0.0-20170606222847-b7d0fc9aadaa
+	go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
+	google.golang.org/genproto v0.0.0-20201008135153-289734e2e40c // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
+)