Use gauges for time metrics (#3009)

Author: StephenButtolph

cache/metercacher/metrics.go

@@ -27,10 +27,10 @@ var (
 
 type metrics struct {
 	getCount *prometheus.CounterVec
-	getTime  *prometheus.CounterVec
+	getTime  *prometheus.GaugeVec
 
 	putCount prometheus.Counter
-	putTime  prometheus.Counter
+	putTime  prometheus.Gauge
 
 	len           prometheus.Gauge
 	portionFilled prometheus.Gauge
@@ -49,8 +49,8 @@ func newMetrics(
 			},
 			resultLabels,
 		),
-		getTime: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		getTime: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Name:      "get_time",
 				Help:      "time spent (ns) in get calls",
@@ -62,7 +62,7 @@ func newMetrics(
 			Name:      "put_count",
 			Help:      "number of put calls",
 		}),
-		putTime: prometheus.NewCounter(prometheus.CounterOpts{
+		putTime: prometheus.NewGauge(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Name:      "put_time",
 			Help:      "time spent (ns) in put calls",

database/leveldb/metrics.go

@@ -19,7 +19,7 @@ type metrics struct {
 	writesDelayedCount prometheus.Counter
 	// total amount of time (in ns) that writes that have been delayed due to
 	// compaction
-	writesDelayedDuration prometheus.Counter
+	writesDelayedDuration prometheus.Gauge
 	// set to 1 if there is currently at least one write that is being delayed
 	// due to compaction
 	writeIsDelayed prometheus.Gauge
@@ -44,7 +44,7 @@ type metrics struct {
 	// size of each level
 	levelSize *prometheus.GaugeVec
 	// amount of time spent compacting each level
-	levelDuration *prometheus.CounterVec
+	levelDuration *prometheus.GaugeVec
 	// amount of bytes read while compacting each level
 	levelReads *prometheus.CounterVec
 	// amount of bytes written while compacting each level
@@ -69,7 +69,7 @@ func newMetrics(namespace string, reg prometheus.Registerer) (metrics, error) {
 			Name:      "writes_delayed",
 			Help:      "number of cumulative writes that have been delayed due to compaction",
 		}),
-		writesDelayedDuration: prometheus.NewCounter(prometheus.CounterOpts{
+		writesDelayedDuration: prometheus.NewGauge(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Name:      "writes_delayed_duration",
 			Help:      "amount of time (in ns) that writes have been delayed due to compaction",
@@ -129,8 +129,8 @@ func newMetrics(namespace string, reg prometheus.Registerer) (metrics, error) {
 			},
 			levelLabels,
 		),
-		levelDuration: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		levelDuration: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Name:      "duration",
 				Help:      "amount of time (in ns) spent in compaction by level",

database/meterdb/db.go

@@ -92,7 +92,7 @@ type Database struct {
 	db database.Database
 
 	calls    *prometheus.CounterVec
-	duration *prometheus.CounterVec
+	duration *prometheus.GaugeVec
 	size     *prometheus.CounterVec
 }
 
@@ -112,8 +112,8 @@ func New(
 			},
 			methodLabels,
 		),
-		duration: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		duration: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Name:      "duration",
 				Help:      "time spent in database calls (ns)",

message/messages.go

@@ -142,7 +142,7 @@ type msgBuilder struct {
 
 	zstdCompressor compression.Compressor
 	count          *prometheus.CounterVec // type + op + direction
-	duration       *prometheus.CounterVec // type + op + direction
+	duration       *prometheus.GaugeVec   // type + op + direction
 
 	maxMessageTimeout time.Duration
 }
@@ -170,8 +170,8 @@ func newMsgBuilder(
 			},
 			metricLabels,
 		),
-		duration: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		duration: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Name:      "compressed_duration",
 				Help:      "time spent handling compressed messages",

network/p2p/network.go

@@ -63,8 +63,8 @@ func NewNetwork(
 	namespace string,
 ) (*Network, error) {
 	metrics := metrics{
-		msgTime: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		msgTime: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Name:      "msg_time",
 				Help:      "message handling time (ns)",

network/p2p/router.go

@@ -45,7 +45,7 @@ type meteredHandler struct {
 }
 
 type metrics struct {
-	msgTime  *prometheus.CounterVec
+	msgTime  *prometheus.GaugeVec
 	msgCount *prometheus.CounterVec
 }
 

snow/networking/handler/metrics.go

@@ -12,8 +12,8 @@ import (
 type metrics struct {
 	expired             *prometheus.CounterVec // op
 	messages            *prometheus.CounterVec // op
-	lockingTime         prometheus.Counter
-	messageHandlingTime *prometheus.CounterVec // op
+	lockingTime         prometheus.Gauge
+	messageHandlingTime *prometheus.GaugeVec // op
 }
 
 func newMetrics(namespace string, reg prometheus.Registerer) (*metrics, error) {
@@ -34,15 +34,15 @@ func newMetrics(namespace string, reg prometheus.Registerer) (*metrics, error) {
 			},
 			opLabels,
 		),
-		messageHandlingTime: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		messageHandlingTime: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Name:      "message_handling_time",
 				Help:      "time spent handling messages",
 			},
 			opLabels,
 		),
-		lockingTime: prometheus.NewCounter(prometheus.CounterOpts{
+		lockingTime: prometheus.NewGauge(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Name:      "locking_time",
 			Help:      "time spent acquiring the context lock",

snow/networking/timeout/metrics.go

@@ -62,7 +62,7 @@ func (m *metrics) Observe(chainID ids.ID, op message.Op, latency time.Duration)
 // chainMetrics contains message response time metrics for a chain
 type chainMetrics struct {
 	messages         *prometheus.CounterVec // op
-	messageLatencies *prometheus.CounterVec // op
+	messageLatencies *prometheus.GaugeVec   // op
 }
 
 func newChainMetrics(reg prometheus.Registerer) (*chainMetrics, error) {
@@ -75,8 +75,8 @@ func newChainMetrics(reg prometheus.Registerer) (*chainMetrics, error) {
 			},
 			opLabels,
 		),
-		messageLatencies: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
+		messageLatencies: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
 				Namespace: responseNamespace,
 				Name:      "message_latencies",
 				Help:      "message latencies (ns)",