Allow the router to trigger metrics capture right before reload

Even on short reload intervals we ensure we capture metrics before
restart. A more sophisticated implementation could use a restart to
detect when counters are reset.

Author: smarterclayton

pkg/cmd/infra/router/template.go

@@ -201,6 +201,7 @@ func (o *TemplateRouterOptions) Validate() error {
 
 // Run launches a template router using the provided options. It never exits.
 func (o *TemplateRouterOptions) Run() error {
+	var reloadCallbacks []func()
 	switch {
 	case o.MetricsType == "haproxy" && len(o.ListenAddr) > 0:
 		var timeout time.Duration
@@ -229,13 +230,17 @@ func (o *TemplateRouterOptions) Run() error {
 				exported = append(exported, i)
 			}
 		}
-		haproxy.NewPrometheusCollector(haproxy.PrometheusOptions{
+		e, err := haproxy.NewPrometheusCollector(haproxy.PrometheusOptions{
 			ScrapeURI:          util.Env("ROUTER_METRICS_HAPROXY_SCRAPE_URI", ""),
 			PidFile:            util.Env("ROUTER_METRICS_HAPROXY_PID_FILE", ""),
 			ServerThreshold:    serverThreshold,
 			BaseScrapeInterval: baseScrapeInterval,
 			ExportedMetrics:    exported,
 		})
+		if err != nil {
+			return err
+		}
+		reloadCallbacks = append(reloadCallbacks, e.CollectNow)
 	}
 	if len(o.ListenAddr) > 0 {
 		metrics.Listen(o.ListenAddr, o.StatsUsername, o.StatsPassword)
@@ -246,6 +251,7 @@ func (o *TemplateRouterOptions) Run() error {
 		TemplatePath:           o.TemplateFile,
 		ReloadScriptPath:       o.ReloadScript,
 		ReloadInterval:         o.ReloadInterval,
+		ReloadCallbacks:        reloadCallbacks,
 		DefaultCertificate:     o.DefaultCertificate,
 		DefaultCertificatePath: o.DefaultCertificatePath,
 		DefaultCertificateDir:  o.DefaultCertificateDir,

pkg/router/template/plugin.go

@@ -41,6 +41,7 @@ type TemplatePluginConfig struct {
 	TemplatePath           string
 	ReloadScriptPath       string
 	ReloadInterval         time.Duration
+	ReloadCallbacks        []func()
 	DefaultCertificate     string
 	DefaultCertificatePath string
 	DefaultCertificateDir  string
@@ -130,6 +131,7 @@ func NewTemplatePlugin(cfg TemplatePluginConfig, lookupSvc ServiceLookup) (*Temp
 		templates:              templates,
 		reloadScriptPath:       cfg.ReloadScriptPath,
 		reloadInterval:         cfg.ReloadInterval,
+		reloadCallbacks:        cfg.ReloadCallbacks,
 		defaultCertificate:     cfg.DefaultCertificate,
 		defaultCertificatePath: cfg.DefaultCertificatePath,
 		defaultCertificateDir:  cfg.DefaultCertificateDir,

pkg/router/template/router.go

@@ -51,6 +51,7 @@ type templateRouter struct {
 	templates        map[string]*template.Template
 	reloadScriptPath string
 	reloadInterval   time.Duration
+	reloadCallbacks  []func()
 	state            map[string]ServiceAliasConfig
 	serviceUnits     map[string]ServiceUnit
 	certManager      certificateManager
@@ -99,6 +100,7 @@ type templateRouterCfg struct {
 	templates              map[string]*template.Template
 	reloadScriptPath       string
 	reloadInterval         time.Duration
+	reloadCallbacks        []func()
 	defaultCertificate     string
 	defaultCertificatePath string
 	defaultCertificateDir  string
@@ -158,6 +160,7 @@ func newTemplateRouter(cfg templateRouterCfg) (*templateRouter, error) {
 		templates:              cfg.templates,
 		reloadScriptPath:       cfg.reloadScriptPath,
 		reloadInterval:         cfg.reloadInterval,
+		reloadCallbacks:        cfg.reloadCallbacks,
 		state:                  make(map[string]ServiceAliasConfig),
 		serviceUnits:           make(map[string]ServiceUnit),
 		certManager:            certManager,
@@ -384,17 +387,25 @@ func (r *templateRouter) Commit() {
 
 // commitAndReload refreshes the backend and persists the router state.
 func (r *templateRouter) commitAndReload() error {
-	r.lock.Lock()
-	defer r.lock.Unlock()
+	// only state changes must be done under the lock
+	if err := func() error {
+		r.lock.Lock()
+		defer r.lock.Unlock()
+
+		glog.V(4).Infof("Writing the router state")
+		if err := r.writeState(); err != nil {
+			return err
+		}
 
-	glog.V(4).Infof("Writing the router state")
-	if err := r.writeState(); err != nil {
+		glog.V(4).Infof("Writing the router config")
+		return r.writeConfig()
+	}(); err != nil {
 		return err
 	}
 
-	glog.V(4).Infof("Writing the router config")
-	if err := r.writeConfig(); err != nil {
-		return err
+	for i, fn := range r.reloadCallbacks {
+		glog.V(4).Infof("Calling reload function %d", i)
+		fn()
 	}
 
 	glog.V(4).Infof("Reloading the router")