instrumentation: add internal metrics server

This commit adds an internal server to the prom-label-proxy that exposes
HTTP metrics about all of the routes registered with the
prom-label-proxy. The internal server also exposes pprof profiles. The
internal server is only activated if the --internal-listen-address flag
is provided.

Signed-off-by: Lucas Servén Marín <[email protected]>

Author: squat

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/efficientgo/tools/core v0.0.0-20220225185207-fe763185946b
 	github.com/go-openapi/runtime v0.24.0
 	github.com/go-openapi/strfmt v0.21.2
+	github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a
+	github.com/oklog/run v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/alertmanager v0.24.0
 	github.com/prometheus/client_golang v1.12.1

go.sum

@@ -77,6 +77,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM=
+github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -754,6 +756,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
+github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a h1:0usWxe5SGXKQovz3p+BiQ81Jy845xSMu2CWKuXsXuUM=
+github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a/go.mod h1:3OETvrxfELvGsU2RoGGWercfeZ4bCL3+SOwzIWtJH/Q=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -805,6 +809,7 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -887,6 +892,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
+github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=

injectproxy/routes.go

@@ -24,8 +24,9 @@ import (
 	"strings"
 
 	"github.com/efficientgo/tools/core/pkg/merrors"
+	"github.com/metalmatze/signal/server/signalhttp"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql/parser"
 )
@@ -41,7 +42,7 @@ type routes struct {
 	label      string
 	labelValue string
 
-	mux            *http.ServeMux
+	mux            http.Handler
 	modifiers      map[string]func(*http.Response) error
 	errorOnReplace bool
 }
@@ -51,6 +52,7 @@ type options struct {
 	enableLabelAPIs  bool
 	passthroughPaths []string
 	errorOnReplace   bool
+	registerer       prometheus.Registerer
 }
 
 type Option interface {
@@ -63,6 +65,13 @@ func (f optionFunc) apply(o *options) {
 	f(o)
 }
 
+// WithPrometheusRegistry configures the proxy to use the given registerer.
+func WithPrometheusRegistry(reg prometheus.Registerer) Option {
+	return optionFunc(func(o *options) {
+		o.registerer = reg
+	})
+}
+
 // WithEnabledLabelsAPI enables proxying to labels API. If false, "501 Not implemented" will be return for those.
 func WithEnabledLabelsAPI() Option {
 	return optionFunc(func(o *options) {
@@ -95,17 +104,22 @@ func WithLabelValue(value string) Option {
 	})
 }
 
+// mux abstracts away the behavior we expect from the http.ServeMux type in this package.
+type mux interface {
+	http.Handler
+	Handle(string, http.Handler)
+}
+
 // strictMux is a mux that wraps standard HTTP handler with safer handler that allows safe user provided handler registrations.
 type strictMux struct {
+	mux
 	seen map[string]struct{}
-
-	m *http.ServeMux
 }
 
-func newStrictMux() *strictMux {
+func newStrictMux(m mux) *strictMux {
 	return &strictMux{
-		seen: map[string]struct{}{},
-		m:    http.NewServeMux(),
+		m,
+		map[string]struct{}{},
 	}
 
 }
@@ -131,18 +145,39 @@ func (s *strictMux) Handle(pattern string, handler http.Handler) error {
 		}
 	}
 
-	s.m.Handle(sanitized, handler)
-	s.m.Handle(sanitized+"/", handler)
+	s.mux.Handle(sanitized, handler)
+	s.mux.Handle(sanitized+"/", handler)
 	s.seen[sanitized] = struct{}{}
 
 	return nil
 }
 
+// instrumentedMux wraps a mux and instruments it.
+type instrumentedMux struct {
+	mux
+	i signalhttp.HandlerInstrumenter
+}
+
+func newInstrumentedMux(m mux, r prometheus.Registerer) *instrumentedMux {
+	return &instrumentedMux{
+		m,
+		signalhttp.NewHandlerInstrumenter(r, []string{"handler"}),
+	}
+}
+
+// Handle implements the mux interface.
+func (i *instrumentedMux) Handle(pattern string, handler http.Handler) {
+	i.mux.Handle(pattern, i.i.NewHandler(prometheus.Labels{"handler": pattern}, handler))
+}
+
 func NewRoutes(upstream *url.URL, label string, opts ...Option) (*routes, error) {
 	opt := options{}
 	for _, o := range opts {
 		o.apply(&opt)
 	}
+	if opt.registerer == nil {
+		opt.registerer = prometheus.NewRegistry()
+	}
 
 	proxy := httputil.NewSingleHostReverseProxy(upstream)
 
@@ -153,7 +188,7 @@ func NewRoutes(upstream *url.URL, label string, opts ...Option) (*routes, error)
 		labelValue:     opt.labelValue,
 		errorOnReplace: opt.errorOnReplace,
 	}
-	mux := newStrictMux()
+	mux := newStrictMux(newInstrumentedMux(http.NewServeMux(), opt.registerer))
 
 	errs := merrors.New(
 		mux.Handle("/federate", r.enforceLabel(enforceMethods(r.matcher, "GET"))),
@@ -185,7 +220,6 @@ func NewRoutes(upstream *url.URL, label string, opts ...Option) (*routes, error)
 		mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			_ = json.NewEncoder(w).Encode(map[string]bool{"ok": true})
 		})),
-		mux.Handle("/metrics", promhttp.Handler()),
 	)
 
 	if err := errs.Err(); err != nil {
@@ -213,7 +247,7 @@ func NewRoutes(upstream *url.URL, label string, opts ...Option) (*routes, error)
 		}
 	}
 
-	r.mux = mux.m
+	r.mux = mux
 	r.modifiers = map[string]func(*http.Response) error{
 		"/api/v1/rules":  modifyAPIResponse(r.filterRules),
 		"/api/v1/alerts": modifyAPIResponse(r.filterAlerts),

main.go

@@ -14,22 +14,29 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"log"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"os/signal"
 	"strings"
 	"syscall"
 
+	"github.com/metalmatze/signal/internalserver"
+	"github.com/oklog/run"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+
 	"github.com/prometheus-community/prom-label-proxy/injectproxy"
 )
 
 func main() {
 	var (
 		insecureListenAddress  string
+		internalListenAddress  string
 		upstream               string
 		label                  string
 		labelValue             string
@@ -40,6 +47,7 @@ func main() {
 
 	flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
 	flagset.StringVar(&insecureListenAddress, "insecure-listen-address", "", "The address the prom-label-proxy HTTP server should listen on.")
+	flagset.StringVar(&internalListenAddress, "internal-listen-address", "", "The address the internal prom-label-proxy HTTP server should listen on to expose metrics about itself.")
 	flagset.StringVar(&upstream, "upstream", "", "The upstream URL to proxy to.")
 	flagset.StringVar(&label, "label", "", "The label to enforce in all proxied PromQL queries. "+
 		"This label will be also required as the URL parameter to get the value to be injected. For example: -label=tenant will"+
@@ -69,7 +77,13 @@ func main() {
 		log.Fatalf("Invalid scheme for upstream URL %q, only 'http' and 'https' are supported", upstream)
 	}
 
-	var opts []injectproxy.Option
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+	)
+
+	opts := []injectproxy.Option{injectproxy.WithPrometheusRegistry(reg)}
 	if enableLabelAPIs {
 		opts = append(opts, injectproxy.WithEnabledLabelsAPI())
 	}
@@ -83,38 +97,69 @@ func main() {
 		opts = append(opts, injectproxy.WithLabelValue(labelValue))
 	}
 
-	routes, err := injectproxy.NewRoutes(upstreamURL, label, opts...)
-	if err != nil {
-		log.Fatalf("Failed to create injectproxy Routes: %v", err)
-	}
+	var g run.Group
+
+	{
+		// Run the insecure HTTP server.
+		routes, err := injectproxy.NewRoutes(upstreamURL, label, opts...)
+		if err != nil {
+			log.Fatalf("Failed to create injectproxy Routes: %v", err)
+		}
 
-	mux := http.NewServeMux()
-	mux.Handle("/", routes)
+		mux := http.NewServeMux()
+		mux.Handle("/", routes)
 
-	srv := &http.Server{Handler: mux}
+		srv := &http.Server{Handler: mux}
 
-	l, err := net.Listen("tcp", insecureListenAddress)
-	if err != nil {
-		log.Fatalf("Failed to listen on insecure address: %v", err)
+		l, err := net.Listen("tcp", insecureListenAddress)
+		if err != nil {
+			log.Fatalf("Failed to listen on insecure address: %v", err)
+		}
+
+		g.Add(func() error {
+			log.Printf("Listening insecurely on %v", l.Addr())
+			if err := srv.Serve(l); err != nil && err != http.ErrServerClosed {
+				log.Printf("Server stopped with %v", err)
+				return err
+			}
+			return nil
+		}, func(error) {
+			l.Close()
+		})
 	}
 
-	errCh := make(chan error)
-	go func() {
-		log.Printf("Listening insecurely on %v", l.Addr())
-		errCh <- srv.Serve(l)
-	}()
-
-	term := make(chan os.Signal, 1)
-	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
-
-	select {
-	case <-term:
-		log.Print("Received SIGTERM, exiting gracefully...")
-		srv.Close()
-	case err := <-errCh:
-		if err != http.ErrServerClosed {
+	if internalListenAddress != "" {
+		// Run the internal HTTP server.
+		h := internalserver.NewHandler(
+			internalserver.WithName("Internal prom-label-proxy API"),
+			internalserver.WithPrometheusRegistry(reg),
+			internalserver.WithPProf(),
+		)
+		// Run the HTTP server.
+		l, err := net.Listen("tcp", internalListenAddress)
+		if err != nil {
+			log.Fatalf("Failed to listen on internal address: %v", err)
+		}
+
+		g.Add(func() error {
+			log.Printf("Listening on %v for metrics and pprof", l.Addr())
+			if err := http.Serve(l, h); err != nil && err != http.ErrServerClosed {
+				log.Printf("Internal server stopped with %v", err)
+				return err
+			}
+			return nil
+		}, func(error) {
+			l.Close()
+		})
+	}
+
+	g.Add(run.SignalHandler(context.Background(), syscall.SIGINT, syscall.SIGTERM))
+
+	if err := g.Run(); err != nil {
+		if !errors.As(err, &run.SignalError{}) {
 			log.Printf("Server stopped with %v", err)
+			os.Exit(1)
 		}
-		os.Exit(1)
+		log.Print("Caught signal; exiting gracefully...")
 	}
 }