Do not fail postgresCollector when version query fails

This no longer returns an error when creating a collector.instance when the database cannot be reached for the version query. This will resolve the entire postgresCollector not being registered for metrics collection when a database is not available. There is now a goroutine that will poll for version information regularly which will exit on os signal. This does not solve for the instance version being blank when collecting metrics.

Signed-off-by: Joe Adams <[email protected]>

Author: sysadmind

Diff for: cmd/postgres_exporter/main.go

@@ -14,10 +14,13 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/go-kit/log"
@@ -132,7 +135,11 @@ func main() {
 		dsn = dsns[0]
 	}
 
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, os.Kill)
+	defer cancel()
+
 	pe, err := collector.NewPostgresCollector(
+		ctx,
 		logger,
 		excludedDatabases,
 		dsn,
@@ -169,8 +176,19 @@ func main() {
 	http.HandleFunc("/probe", handleProbe(logger, excludedDatabases))
 
 	srv := &http.Server{}
+
+	go func() {
+		<-ctx.Done()
+		level.Info(logger).Log("msg", "Shutting down server...")
+		if err := srv.Shutdown(context.Background()); err != nil {
+			level.Error(logger).Log("msg", "Error shutting down server", "err", err)
+		}
+	}()
+
 	if err := web.ListenAndServe(srv, webConfig, logger); err != nil {
-		level.Error(logger).Log("msg", "Error running HTTP server", "err", err)
-		os.Exit(1)
+		if err != http.ErrServerClosed {
+			level.Error(logger).Log("msg", "Error running HTTP server", "err", err)
+			os.Exit(1)
+		}
 	}
 }

Diff for: collector/collector.go

@@ -97,7 +97,7 @@ type PostgresCollector struct {
 type Option func(*PostgresCollector) error
 
 // NewPostgresCollector creates a new PostgresCollector.
-func NewPostgresCollector(logger log.Logger, excludeDatabases []string, dsn string, filters []string, options ...Option) (*PostgresCollector, error) {
+func NewPostgresCollector(ctx context.Context, logger log.Logger, excludeDatabases []string, dsn string, filters []string, options ...Option) (*PostgresCollector, error) {
 	p := &PostgresCollector{
 		logger: logger,
 	}
@@ -148,7 +148,7 @@ func NewPostgresCollector(logger log.Logger, excludeDatabases []string, dsn stri
 		return nil, errors.New("empty dsn")
 	}
 
-	instance, err := newInstance(dsn)
+	instance, err := newCollectorInstance(ctx, dsn, logger)
 	if err != nil {
 		return nil, err
 	}

Diff for: collector/instance.go

@@ -14,19 +14,59 @@
 package collector
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"regexp"
+	"time"
 
 	"github.com/blang/semver/v4"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+)
+
+var (
+	versionPollInterval = 5 * time.Minute
 )
 
 type instance struct {
 	db      *sql.DB
 	version semver.Version
 }
 
-func newInstance(dsn string) (*instance, error) {
+func newProbeInstance(dsn string, logger log.Logger) (*instance, error) {
+	return newInstance(dsn, logger)
+}
+
+func newCollectorInstance(ctx context.Context, dsn string, logger log.Logger) (*instance, error) {
+	i, err := newInstance(dsn, logger)
+	if err != nil {
+		return nil, err
+	}
+
+	// Poll for updated version information
+	go func() {
+		ticker := time.NewTicker(versionPollInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				version, err := queryVersion(i.getDB())
+				if err != nil {
+					level.Warn(logger).Log("msg", "Error querying postgresql version", "err", err)
+				} else {
+					i.version = version
+				}
+			}
+		}
+	}()
+
+	return i, nil
+}
+
+func newInstance(dsn string, logger log.Logger) (*instance, error) {
 	i := &instance{}
 	db, err := sql.Open("postgres", dsn)
 	if err != nil {
@@ -38,12 +78,10 @@ func newInstance(dsn string) (*instance, error) {
 
 	version, err := queryVersion(db)
 	if err != nil {
-		db.Close()
-		return nil, err
+		level.Warn(logger).Log("msg", "Error querying postgresql version", "err", err)
+	} else {
+		i.version = version
 	}
-
-	i.version = version
-
 	return i, nil
 }
 

Diff for: collector/probe.go

@@ -57,7 +57,7 @@ func NewProbeCollector(logger log.Logger, excludeDatabases []string, registry *p
 		}
 	}
 
-	instance, err := newInstance(dsn.GetConnectionString())
+	instance, err := newProbeInstance(dsn.GetConnectionString(), logger)
 	if err != nil {
 		return nil, err
 	}