Expose prometheus metrics for the router by default

Add new settings to the router that allow metrics to be captured via
/metrics on a configurable listen address. Metrics and pprof data are
only available when stats username and password are set, and require
auth.

The metrics implementation uses a variant of prometheus/haproxy_exporter
and attempts to minimize the amount of metrics returned for the router.
It also rate limits the frequency at which the haproxy stats endpoint is
scraped relative to the amount of servers defined (a rough rule of thumb
from this data is that for every 1000 servers the minimum interval
increases by 5 seconds). The metrics returned are not transformed to be
service based, but could be.

This currently generates ~1.8M per 1k routes. Shorter server identifiers
would have an impact. This generates a ~2% CPU cost on the router
endpoint when run every 2s, which should scale reasonably to
infrastructure.

Author: smarterclayton

Diff for: pkg/cmd/admin/router/router.go

@@ -678,6 +678,16 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
 		}
 		env["ROUTER_CANONICAL_HOSTNAME"] = cfg.RouterCanonicalHostname
 	}
+	// automatically start the internal metrics agent if the type has metrics
+	if cfg.Type == "haproxy-router" {
+		env["ROUTER_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", defaultStatsPort-1)
+		env["ROUTER_METRICS_TYPE"] = "haproxy"
+		ports = append(ports, kapi.ContainerPort{
+			Name:          "router-stats",
+			ContainerPort: int32(defaultStatsPort - 1),
+			Protocol:      kapi.ProtocolTCP,
+		})
+	}
 	env.Add(secretEnv)
 	if len(defaultCert) > 0 {
 		if cfg.SecretsAsEnv {

Diff for: pkg/cmd/infra/router/router.go

@@ -56,6 +56,8 @@ type RouterSelection struct {
 	DisableNamespaceOwnershipCheck bool
 
 	EnableIngress bool
+
+	ListenAddr string
 }
 
 // Bind sets the appropriate labels
@@ -73,6 +75,7 @@ func (o *RouterSelection) Bind(flag *pflag.FlagSet) {
 	flag.BoolVar(&o.AllowWildcardRoutes, "allow-wildcard-routes", cmdutil.Env("ROUTER_ALLOW_WILDCARD_ROUTES", "") == "true", "Allow wildcard host names for routes")
 	flag.BoolVar(&o.DisableNamespaceOwnershipCheck, "disable-namespace-ownership-check", cmdutil.Env("ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK", "") == "true", "Disables the namespace ownership checks for a route host with different paths or for overlapping host names in the case of wildcard routes. Please be aware that if namespace ownership checks are disabled, routes in a different namespace can use this mechanism to 'steal' sub-paths for existing domains. This is only safe if route creation privileges are restricted, or if all the users can be trusted.")
 	flag.BoolVar(&o.EnableIngress, "enable-ingress", cmdutil.Env("ROUTER_ENABLE_INGRESS", "") == "true", "Enable configuration via ingress resources")
+	flag.StringVar(&o.ListenAddr, "listen-addr", cmdutil.Env("ROUTER_LISTEN_ADDR", ""), "The name of an interface to listen on to expose metrics and health checking. If not specified, will not listen.")
 }
 
 // RouteSelectionFunc returns a func that identifies the host for a route.

Diff for: pkg/cmd/infra/router/template.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/golang/glog"
@@ -21,6 +22,8 @@ import (
 	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
 	"github.com/openshift/origin/pkg/router"
 	"github.com/openshift/origin/pkg/router/controller"
+	"github.com/openshift/origin/pkg/router/metrics"
+	"github.com/openshift/origin/pkg/router/metrics/haproxy"
 	templateplugin "github.com/openshift/origin/pkg/router/template"
 	"github.com/openshift/origin/pkg/util/proc"
 )
@@ -66,6 +69,7 @@ type TemplateRouter struct {
 	RouterService           *ktypes.NamespacedName
 	BindPortsAfterSync      bool
 	MaxConnections          string
+	MetricsType             string
 }
 
 // reloadInterval returns how often to run the router reloads. The interval
@@ -93,6 +97,7 @@ func (o *TemplateRouter) Bind(flag *pflag.FlagSet) {
 	flag.BoolVar(&o.ExtendedValidation, "extended-validation", util.Env("EXTENDED_VALIDATION", "true") == "true", "If set, then an additional extended validation step is performed on all routes admitted in by this router. Defaults to true and enables the extended validation checks.")
 	flag.BoolVar(&o.BindPortsAfterSync, "bind-ports-after-sync", util.Env("ROUTER_BIND_PORTS_AFTER_SYNC", "") == "true", "Bind ports only after route state has been synchronized")
 	flag.StringVar(&o.MaxConnections, "max-connections", util.Env("ROUTER_MAX_CONNECTIONS", ""), "Specifies the maximum number of concurrent connections.")
+	flag.StringVar(&o.MetricsType, "metrics-type", util.Env("ROUTER_METRICS_TYPE", ""), "Specifies the type of metrics to gather. Supports 'haproxy'.")
 }
 
 type RouterStats struct {
@@ -178,6 +183,9 @@ func (o *TemplateRouterOptions) Complete() error {
 }
 
 func (o *TemplateRouterOptions) Validate() error {
+	if len(o.MetricsType) > 0 && o.MetricsType != "haproxy" {
+		return errors.New("supported metrics types are: 'haproxy'")
+	}
 	if len(o.RouterName) == 0 {
 		return errors.New("router must have a name to identify itself in route status")
 	}
@@ -193,6 +201,46 @@ func (o *TemplateRouterOptions) Validate() error {
 
 // Run launches a template router using the provided options. It never exits.
 func (o *TemplateRouterOptions) Run() error {
+	switch {
+	case o.MetricsType == "haproxy" && len(o.ListenAddr) > 0:
+		var timeout time.Duration
+		if t := util.Env("ROUTER_METRICS_HAPROXY_TIMEOUT", ""); len(t) > 0 {
+			d, err := time.ParseDuration(t)
+			if err != nil {
+				return fmt.Errorf("ROUTER_METRICS_HAPROXY_TIMEOUT is not a valid duration: %v", err)
+			}
+			timeout = d
+		}
+		var baseScrapeInterval time.Duration
+		if t := util.Env("ROUTER_METRICS_HAPROXY_BASE_SCRAPE_INTERVAL", ""); len(t) > 0 {
+			d, err := time.ParseDuration(t)
+			if err != nil {
+				return fmt.Errorf("ROUTER_METRICS_HAPROXY_BASE_SCRAPE_INTERVAL is not a valid duration: %v", err)
+			}
+			baseScrapeInterval = d
+		}
+		var exported []int
+		if t := util.Env("ROUTER_METRICS_HAPROXY_EXPORTED", ""); len(t) > 0 {
+			for _, s := range strings.Split(t, ",") {
+				i, err := strconv.Atoi(s)
+				if err != nil {
+					return errors.New("ROUTER_METRICS_HAPROXY_EXPORTED must be a comma delimited list of non-negative integers")
+				}
+				exported = append(exported, i)
+			}
+		}
+		haproxy.NewPrometheusCollector(haproxy.PrometheusOptions{
+			ScrapeURI:          util.Env("ROUTER_METRICS_HAPROXY_SCRAPE_URI", ""),
+			PidFile:            util.Env("ROUTER_METRICS_HAPROXY_PID_FILE", ""),
+			ServerThreshold:    serverThreshold,
+			BaseScrapeInterval: baseScrapeInterval,
+			ExportedMetrics:    exported,
+		})
+	}
+	if len(o.ListenAddr) > 0 {
+		metrics.Listen(o.ListenAddr, o.StatsUsername, o.StatsPassword)
+	}
+
 	pluginCfg := templateplugin.TemplatePluginConfig{
 		WorkingDir:             o.WorkingDir,
 		TemplatePath:           o.TemplateFile,