Add rate limiting per tenant (observatorium#102)

* Implement rate limiting per tenant

Signed-off-by: Kemal Akkoyun <[email protected]>

* Add instrumentation for grpc client

Signed-off-by: Kemal Akkoyun <[email protected]>

* Draw finer package boundaries

Signed-off-by: Kemal Akkoyun <[email protected]>

* Fix protoc install script

Signed-off-by: Kemal Akkoyun <[email protected]>

Author: kakkoyun

Diff for: .bingo/.gitignore

@@ -1,5 +1,5 @@
 
-# Ignore everything
+# Ignore everything.
 *
 
 # But not these files:

Diff for: .bingo/Variables.mk

@@ -82,3 +82,15 @@ $(UP): .bingo/up.mod
 	@echo "(re)installing $(GOBIN)/up-v0.0.0-20200928171403-120d85735d11"
 	@cd .bingo && $(GO) build -modfile=up.mod -o=$(GOBIN)/up-v0.0.0-20200928171403-120d85735d11 "github.com/observatorium/up/cmd/up"
 
+GUBERNATOR := $(GOBIN)/gubernator-v0.8.4-0.20200617200142-07e238f8cd86
+$(GUBERNATOR): .bingo/gubernator.mod
+	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
+	@echo "(re)installing $(GOBIN)/gubernator-v0.8.4-0.20200617200142-07e238f8cd86"
+	@cd .bingo && $(GO) build -modfile=gubernator.mod -o=$(GOBIN)/gubernator-v0.8.4-0.20200617200142-07e238f8cd86 "github.com/mailgun/gubernator/cmd/gubernator"
+
+PROTOC_GEN_GO := $(GOBIN)/protoc-gen-go-v1.4.2
+$(PROTOC_GEN_GO): .bingo/protoc-gen-go.mod
+	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
+	@echo "(re)installing $(GOBIN)/protoc-gen-go-v1.4.2"
+	@cd .bingo && $(GO) build -modfile=protoc-gen-go.mod -o=$(GOBIN)/protoc-gen-go-v1.4.2 "github.com/golang/protobuf/protoc-gen-go"
+

Diff for: .bingo/gubernator.mod

@@ -0,0 +1,5 @@
+module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
+
+go 1.15
+
+require github.com/mailgun/gubernator v0.8.4-0.20200617200142-07e238f8cd86 // cmd/gubernator

Diff for: .bingo/protoc-gen-go.mod

@@ -0,0 +1,5 @@
+module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
+
+go 1.15
+
+require github.com/golang/protobuf v1.4.2 // protoc-gen-go

Diff for: .bingo/variables.env

@@ -30,3 +30,7 @@ STYX="${gobin}/styx-v0.0.0-20200109161911-78a77eb717b4"
 
 UP="${gobin}/up-v0.0.0-20200928171403-120d85735d11"
 
+GUBERNATOR="${gobin}/gubernator-v0.8.4-0.20200617200142-07e238f8cd86"
+
+PROTOC_GEN_GO="${gobin}/protoc-gen-go-v1.4.2"
+

Diff for: .circleci/config.yml

@@ -35,6 +35,8 @@ jobs:
       - run: |
           make README.md
           make generate validate --always-make
+          apt-get update && apt-get -y install unzip
+          make proto
           git diff --exit-code
 
   container-push:

Diff for: Makefile

@@ -43,6 +43,10 @@ PROMREMOTEBENCH_VERSION ?= 0.8.0
 SHELLCHECK ?= $(BIN_DIR)/shellcheck
 MOCKPROVIDER ?= $(BIN_DIR)/mockprovider
 GENERATE_TLS_CERT ?= $(BIN_DIR)/generate-tls-cert
+
+PROTOC ?= $(TMP_DIR)/protoc
+PROTOC_VERSION ?= 3.13.0
+
 SERVER_CERT ?= $(CERT_DIR)/server.pem
 
 default: observatorium
@@ -88,7 +92,7 @@ validate: $(KUBEVAL)
 
 .PHONY: shellcheck
 shellcheck: $(SHELLCHECK)
-	$(SHELLCHECK) $(shell find . -type f -name "*.sh" -not -path "*vendor*" -not -path "${TMP_DIR}/*")
+	$(SHELLCHECK) $(shell find . -type f -name "*.sh" -not -path "*vendor*" -not -path "*tmp*")
 
 .PHONY: lint
 lint: $(GOLANGCI_LINT) deps shellcheck jsonnet-fmt
@@ -113,8 +117,23 @@ test-load: build load-test-dependencies
 clean:
 	-rm tmp/help.txt
 	-rm -rf tmp/bin
+	-rm -rf tmp/src
 	-rm observatorium
 
+ratelimit/gubernator/proto/google:
+	mkdir -p $(TMP_DIR)/src/grpc-gateway
+	mkdir -p $(shell dirname $@)
+	curl -L "https://github.com/grpc-ecosystem/grpc-gateway/archive/master.tar.gz" | tar --strip-components=1 -xzf - -C $(TMP_DIR)/src/grpc-gateway
+	mv $(TMP_DIR)/src/grpc-gateway/third_party/googleapis/google $@
+
+ratelimit/gubernator/gubernator.proto:
+	curl -L -o ratelimit/gubernator/gubernator.proto "https://raw.githubusercontent.com/mailgun/gubernator/master/proto/gubernator.proto"
+
+.PHONY: proto
+proto: ratelimit/gubernator/proto/google ratelimit/gubernator/gubernator.proto $(PROTOC) $(PROTOC_GEN_GO) $(BIN_DIR)
+	@cp -f $(PROTOC_GEN_GO) $(BIN_DIR)/protoc-gen-go
+	PATH=$$PATH:$(BIN_DIR):$(FIRST_GOPATH)/bin scripts/generate_proto.sh
+
 .PHONY: container
 container: Dockerfile
 	@docker build --build-arg BUILD_DATE="$(BUILD_TIMESTAMP)" \
@@ -139,7 +158,7 @@ container-release: container
 	docker push $(DOCKER_REPO):latest
 
 .PHONY: integration-test-dependencies
-integration-test-dependencies: $(THANOS) $(UP) $(DEX) $(LOKI) $(WEBSOCAT) $(OPA)
+integration-test-dependencies: $(THANOS) $(UP) $(DEX) $(LOKI) $(WEBSOCAT) $(OPA) $(GUBERNATOR)
 
 .PHONY: load-test-dependencies
 load-test-dependencies: $(PROMREMOTEBENCH) $(PROMETHEUS) $(STYX) $(MOCKPROVIDER)
@@ -184,24 +203,27 @@ $(WEBSOCAT): | $(BIN_DIR)
 	mv $(WEBSOCAT_PKG) websocat && \
 	chmod u+x websocat
 
-$(PROMREMOTEBENCH): | $(BIN_DIR)
-	mkdir -p $(TMP_DIR)/promremotebench
-	curl -L https://github.com/m3dbx/promremotebench/archive/v$(PROMREMOTEBENCH_VERSION).tar.gz | tar --strip-components=1 -xzf - -C $(TMP_DIR)/promremotebench
-	cd $(TMP_DIR)/promremotebench/src && \
+$(PROMREMOTEBENCH): | deps $(BIN_DIR)
+	mkdir -p $(TMP_DIR)/src/promremotebench
+	curl -L https://github.com/m3dbx/promremotebench/archive/v$(PROMREMOTEBENCH_VERSION).tar.gz | tar --strip-components=1 -xzf - -C $(TMP_DIR)/src/promremotebench
+	cd $(TMP_DIR)/src/promremotebench && \
 		go build ./cmd/promremotebench
-	mv $(TMP_DIR)/promremotebench/src/promremotebench $@
+	mv $(TMP_DIR)/src/promremotebench/promremotebench $@
 
 $(SHELLCHECK): $(BIN_DIR)
 	curl -sNL "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.$(OS).$(ARCH).tar.xz" | tar --strip-components=1 -xJf - -C $(BIN_DIR)
 
 $(MOCKPROVIDER): | deps $(BIN_DIR)
-	go build  -tags tools -o $@ github.com/observatorium/observatorium/test/mock
+	go build -tags tools -o $@ github.com/observatorium/observatorium/test/mock
 
 $(GENERATE_TLS_CERT): | deps $(BIN_DIR)
 	# A thin wrapper around github.com/cloudflare/cfssl
 	go build -tags tools -o $@ github.com/observatorium/observatorium/test/tls
 
-# Jsonnet and Example manifests
+$(PROTOC): $(TMP_DIR) $(BIN_DIR)
+	@PROTOC_VERSION="$(PROTOC_VERSION)" TMP_DIR="$(TMP_DIR)" BIN_DIR="$(BIN_DIR)" scripts/install_protoc.sh
+
+# Jsonnet and Example manifests.
 
 CONTAINER_CMD:=docker run --rm \
 		-u="$(shell id -u):$(shell id -g)" \

Diff for: README.md

@@ -75,6 +75,8 @@ Usage of ./observatorium:
     	The name of the HTTP header containing the tenant ID to forward to the metrics upstreams. (default "THANOS-TENANT")
   -metrics.write.endpoint string
     	The endpoint against which to make write requests for metrics.
+  -middleware.rate-limiter.grpc-address string
+    	The gRPC Server Address against which to run rate limit checks when the rate limits are specified for a given tenant. If it is not specified local non-shared rate limiting will be used.
   -rbac.config string
     	Path to the RBAC configuration file. (default "rbac.yaml")
   -tenants.config string

Diff for: api/logs/v1/http.go

@@ -29,7 +29,7 @@ type handlerConfiguration struct {
 	writeMiddlewares []func(http.Handler) http.Handler
 }
 
-// HandlerOption modifies the handler's configuration
+// HandlerOption modifies the handler's configuration.
 type HandlerOption func(h *handlerConfiguration)
 
 // Logger add a custom logger for the handler to use.

Diff for: api/metrics/v1/http.go

@@ -27,7 +27,7 @@ type handlerConfiguration struct {
 	writeMiddlewares []func(http.Handler) http.Handler
 }
 
-// HandlerOption modifies the handler's configuration
+// HandlerOption modifies the handler's configuration.
 type HandlerOption func(h *handlerConfiguration)
 
 // Logger add a custom logger for the handler to use.
@@ -75,7 +75,7 @@ func (n nopInstrumentHandler) NewHandler(labels prometheus.Labels, handler http.
 	return handler.ServeHTTP
 }
 
-// NewHandler creates the new metrics v1 handler
+// NewHandler creates the new metrics v1 handler.
 func NewHandler(read, write *url.URL, opts ...HandlerOption) http.Handler {
 	c := &handlerConfiguration{
 		logger:     log.NewNopLogger(),

Diff for: go.mod

@@ -8,8 +8,12 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-chi/chi v4.0.2+incompatible
+	github.com/go-chi/httprate v0.4.0
 	github.com/go-kit/kit v0.10.0
-	github.com/golang/protobuf v1.4.0 // indirect
+	github.com/golang/protobuf v1.4.2
+	github.com/google/go-cmp v0.5.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/lib/pq v1.3.0 // indirect
 	github.com/mattn/go-sqlite3 v1.11.0 // indirect
 	github.com/metalmatze/signal v0.0.0-20201002154727-d0c16e42a3cf
@@ -21,11 +25,16 @@ require (
 	github.com/prometheus/procfs v0.0.11 // indirect
 	github.com/prometheus/prometheus v1.8.2-0.20200305080338-7164b58945bb
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/stretchr/testify v1.6.1 // indirect
 	go.uber.org/automaxprocs v1.2.0
-	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
+	golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect
+	golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 // indirect
 	google.golang.org/appengine v1.6.1 // indirect
+	google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36
+	google.golang.org/grpc v1.27.1
+	google.golang.org/protobuf v1.23.0
 	gopkg.in/square/go-jose.v2 v2.4.1 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	k8s.io/component-base v0.18.0
 )