Support multi-module releases in go-control-plane (#714)

Support multi-module releases in go-control-plane.
From this point onward, the `go-control-plane` repository is released as multiple packages:
 - `github.com/envoyproxy/go-control-plane/envoy` and `github.com/envoyproxy/go-control-plane/contrib` are released with a version ~ matching  `github.com/envoyproxy/envoy` and should be imported to use generated go protobuf schemas
 - `github.com/envoyproxy/go-control-plane` is only versioning the `go-control-plane` core components (i.e. cache and server)
 - `github.com/envoyproxy/go-control-plane/ratelimit` and `github.com/envoyproxy/go-control-plane/xdsmatcher` are independent features versioned under their own tag

Signed-off-by: Alec Holmes <[email protected]>
Signed-off-by: Valerian Roche <[email protected]>
Signed-off-by: Matthieu MOREL <[email protected]>

Author: alecholmez

.gitignore

@@ -16,3 +16,5 @@ examples/dyplomat/dyplomat
 *_gen_test.go
 
 /envoy*.log
+
+.tools/

CONTRIBUTING.md

@@ -197,4 +197,27 @@ re-run all the CI tasks. Consider adding an alias into your `.gitconfig` file:
 ```
 
 Once you add this alias you can issue the command `git kick-ci` and the PR
-will be sent back for a retest.
+will be sent back for a retest.
+
+# Releasing
+
+The repository is split in multiple packages, themselves using distinct versions:
+ - envoy API
+    - `github.com/envoyproxy/go-control-plane/envoy`, including envoy API generated protobuf files
+    - `github.com/envoyproxy/go-control-plane/contrib`, including envoy API contrib generated protobuf files
+ - ratelimit
+    - `github.com/envoyproxy/go-control-plane/ratelimit`, including generated files for envoy RLS (https://github.com/envoyproxy/ratelimit)
+ - go-control-plane
+    - `github.com/envoyproxy/go-control-plane`
+    - `github.com/envoyproxy/go-control-plane/xdsmatcher`
+
+To create a new release, from a clean branch:
+ - update the version for the desired part in `versions.yaml` and commit
+ - run the following command to update all mod files
+```
+make multimod/prerelease MODSET={part to release}
+```
+ - push the resulting branch to get merged in, then create tags with 
+ ```
+make multimod/push-tags MODSET={part to release}
+ ```

Makefile

@@ -8,9 +8,14 @@ SHELL 	:= /bin/bash
 BINDIR	:= bin
 PKG 	:= github.com/envoyproxy/go-control-plane
 
+include ./Makefile.common
+
 .PHONY: build
 build:
-	@go build ./pkg/... ./envoy/...
+	@make -C pkg common/build
+	@make -C envoy common/build
+	@make -C ratelimit common/build
+	@make -C xdsmatcher common/build 
 
 .PHONY: clean
 clean:
@@ -28,7 +33,7 @@ test:
 
 .PHONY: cover
 cover:
-	@build/coverage.sh
+	@scripts/coverage.sh
 
 .PHONY: examples
 examples:
@@ -58,22 +63,22 @@ $(BINDIR)/test:
 integration: integration.xds integration.ads integration.rest integration.xds.mux integration.xds.delta integration.ads.delta
 
 integration.ads: $(BINDIR)/test $(BINDIR)/upstream
-	env XDS=ads build/integration.sh
+	env XDS=ads scripts/integration.sh
 
 integration.xds: $(BINDIR)/test $(BINDIR)/upstream
-	env XDS=xds build/integration.sh
+	env XDS=xds scripts/integration.sh
 
 integration.rest: $(BINDIR)/test $(BINDIR)/upstream
-	env XDS=rest build/integration.sh
+	env XDS=rest scripts/integration.sh
 
 integration.xds.mux: $(BINDIR)/test $(BINDIR)/upstream
-	env XDS=xds build/integration.sh -mux
+	env XDS=xds scripts/integration.sh -mux
 
 integration.xds.delta: $(BINDIR)/test $(BINDIR)/upstream
-	env XDS=delta build/integration.sh
+	env XDS=delta scripts/integration.sh
 
 integration.ads.delta: $(BINDIR)/test $(BINDIR)/upstream
-	env XDS=delta-ads build/integration.sh
+	env XDS=delta-ads scripts/integration.sh
 
 #--------------------------------------
 #-- example xDS control plane server
@@ -84,15 +89,44 @@ $(BINDIR)/example:
 	@go build -race -o $@ internal/example/main/main.go
 
 example: $(BINDIR)/example
-	@build/example.sh
+	@scripts/example.sh
 
 .PHONY: docker_tests
 docker_tests:
 	docker build --pull -f Dockerfile.ci . -t gcp_ci && \
-	docker run -v $$(pwd):/go-control-plane $$(tty -s && echo "-it" || echo) gcp_ci /bin/bash -c /go-control-plane/build/do_ci.sh
+	docker run -v $$(pwd):/go-control-plane $$(tty -s && echo "-it" || echo) gcp_ci /bin/bash -c /go-control-plane/scripts/do_ci.sh
 
 .PHONY: tidy-all
-tidy-all:
-	go mod tidy
-	make -C examples/dyplomat tidy
-	make -C xdsmatcher tidy
+tidy-all: common/tidy
+	make -C contrib common/tidy
+	make -C envoy common/tidy
+	make -C examples/dyplomat common/tidy
+	make -C ratelimit common/tidy
+	make -C xdsmatcher common/tidy
+
+.PHONY: multimod/verify
+multimod/verify: $(MULTIMOD)
+	@echo "Validating versions.yaml"
+	$(MULTIMOD) verify
+
+.PHONY: multimod/prerelease
+multimod/prerelease: $(MULTIMOD)
+	$(MULTIMOD) prerelease -s=true -b=false -v ./versions.yaml -m ${MODSET}
+	$(MAKE) tidy-all
+
+COMMIT?=HEAD
+REMOTE?[email protected]:envoyproxy/go-control-plane.git
+.PHONY: multimod/tag
+multimod/tag: $(MULTIMOD)
+	$(MULTIMOD) verify
+	$(MULTIMOD) tag -m ${MODSET} -c ${COMMIT} --print-tags
+
+COMMIT?=HEAD
+REMOTE?[email protected]:envoyproxy/go-control-plane.git
+.PHONY: push-tags
+multimod/push-tags: $(MULTIMOD)
+	$(MULTIMOD) verify
+	set -e; for tag in `$(MULTIMOD) tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
+		echo "pushing tag $${tag}"; \
+		git push ${REMOTE} $${tag}; \
+	done;

Makefile.common

@@ -0,0 +1,39 @@
+SHELL = /bin/bash
+.SHELLFLAGS = -o pipefail -c
+
+SRC_ROOT := $(shell git rev-parse --show-toplevel)
+
+GOCMD?= go
+
+TOOLS_MOD_DIR    := $(SRC_ROOT)/internal/tools
+TOOLS_MOD_REGEX  := "\s+_\s+\".*\""
+TOOLS_PKG_NAMES  := $(shell grep -E $(TOOLS_MOD_REGEX) < $(TOOLS_MOD_DIR)/tools.go | tr -d " _\"")
+TOOLS_BIN_DIR    := $(SRC_ROOT)/.tools
+TOOLS_BIN_NAMES  := $(addprefix $(TOOLS_BIN_DIR)/, $(notdir $(TOOLS_PKG_NAMES)))
+
+$(TOOLS_BIN_DIR):
+	mkdir -p $@
+
+$(TOOLS_BIN_NAMES): $(TOOLS_BIN_DIR) $(TOOLS_MOD_DIR)/go.mod
+	cd $(TOOLS_MOD_DIR) && GOOS="" GOARCH="" $(GOCMD) build -o $@ -trimpath $(filter %/$(notdir $@),$(TOOLS_PKG_NAMES))
+
+MULTIMOD            := $(TOOLS_BIN_DIR)/multimod
+GOLANGCI_LINT       := $(TOOLS_BIN_DIR)/golangci-lint
+
+.PHONY: common/build
+common/build:
+	@go build ./...
+
+.PHONY: common/coverage
+common/coverage:
+	$(GOCMD) test ./... -race -covermode=atomic -coverprofile=coverage.out
+common/coverage_html: common/coverage
+	$(GOCMD) tool cover -html=coverage.out
+
+.PHONY: common/test
+common/test:
+	$(GOCMD) test ./... -race
+
+.PHONY: common/tidy
+common/tidy:
+	$(GOCMD) mod tidy

contrib/Makefile

@@ -0,0 +1 @@
+include ../Makefile.common

contrib/go.mod

@@ -0,0 +1,23 @@
+module github.com/envoyproxy/go-control-plane/contrib
+
+go 1.21
+
+replace github.com/envoyproxy/go-control-plane/envoy => ../envoy
+
+require (
+	github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20
+	github.com/envoyproxy/go-control-plane/envoy v0.0.0-00010101000000-000000000000
+	github.com/envoyproxy/protoc-gen-validate v1.1.0
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10
+	google.golang.org/grpc v1.67.1
+	google.golang.org/protobuf v1.35.2
+)
+
+require (
+	cel.dev/expr v0.16.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
+)

contrib/go.sum

@@ -0,0 +1,24 @@
+cel.dev/expr v0.16.0 h1:yloc84fytn4zmJX2GU3TkXGsaieaV7dQ057Qs4sIG2Y=
+cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg=
+github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 h1:N+3sFI5GUjRKBi+i0TxYVST9h4Ie192jJWpHvthBBgg=
+github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
+github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=
+github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8=
+google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
+google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
+google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

envoy/Makefile

@@ -0,0 +1 @@
+include ../Makefile.common

envoy/go.mod

@@ -0,0 +1,22 @@
+module github.com/envoyproxy/go-control-plane/envoy
+
+go 1.21
+
+require (
+	github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20
+	github.com/envoyproxy/protoc-gen-validate v1.1.0
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10
+	github.com/prometheus/client_model v0.6.0
+	go.opentelemetry.io/proto/otlp v1.0.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142
+	google.golang.org/grpc v1.67.1
+	google.golang.org/protobuf v1.35.2
+)
+
+require (
+	cel.dev/expr v0.16.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+)

envoy/go.sum

@@ -0,0 +1,28 @@
+cel.dev/expr v0.16.0 h1:yloc84fytn4zmJX2GU3TkXGsaieaV7dQ057Qs4sIG2Y=
+cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg=
+github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 h1:N+3sFI5GUjRKBi+i0TxYVST9h4Ie192jJWpHvthBBgg=
+github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
+github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=
+github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
+github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
+github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
+go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
+go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8=
+google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
+google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
+google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

examples/dyplomat/Makefile

@@ -1,12 +1 @@
-.PHONY: test
-test:
-	go test ./... -race
-.PHONY: coverage
-coverage:
-	go test ./... -race -covermode=atomic -coverprofile=coverage.out
-coverage_html: coverage
-	go tool cover -html=coverage.out
-
-.PHONY: tidy
-tidy:
-	go mod tidy
+include ../../Makefile.common

examples/dyplomat/go.mod

@@ -2,8 +2,15 @@ module github.com/envoyproxy/go-control-plane/examples/dyplomat
 
 go 1.22.8
 
+replace (
+	github.com/envoyproxy/go-control-plane => ../..
+	github.com/envoyproxy/go-control-plane/envoy => ../../envoy
+	github.com/envoyproxy/go-control-plane/ratelimit => ../../ratelimit
+)
+
 require (
 	github.com/envoyproxy/go-control-plane v0.13.0
+	github.com/envoyproxy/go-control-plane/envoy v0.0.0-00010101000000-000000000000
 	google.golang.org/grpc v1.67.1
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.28.2
@@ -20,6 +27,7 @@ require (
 	github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/envoyproxy/go-control-plane/ratelimit v0.0.0-00010101000000-000000000000 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
@@ -54,7 +62,7 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
@@ -64,5 +72,3 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
-
-replace github.com/envoyproxy/go-control-plane => ../..