Skip to content

Commit 7139569

Browse files
Add reason label to total_reload metric
1 parent 679fc80 commit 7139569

File tree

5 files changed

+48
-33
lines changed

5 files changed

+48
-33
lines changed

docs-web/logging-and-monitoring/prometheus.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ The Ingress Controller exports the following metrics:
2626
* NGINX/NGINX Plus metrics. Please see this [doc](https://github.com/nginxinc/nginx-prometheus-exporter#exported-metrics) to find more information about the exported metrics.
2727

2828
* Ingress Controller metrics
29-
* `controller_nginx_reloads_total`. Number of successful NGINX reloads.
29+
* `controller_nginx_reloads_total`. Number of successful NGINX reloads. This includes the label `reason` with 2 possible values `endpoints` (the reason for the reload was an endpoints update) and `other` (the reload was caused by something other than an endpoint update like an ingress update).
3030
* `controller_nginx_reload_errors_total`. Number of unsuccessful NGINX reloads.
3131
* `controller_nginx_last_reload_status`. Status of the last NGINX reload, 0 meaning down and 1 up.
3232
* `controller_nginx_last_reload_milliseconds`. Duration in milliseconds of the last NGINX reload.

internal/configs/configurator.go

+22-22
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ func (cnf *Configurator) AddOrUpdateIngress(ingEx *IngressEx) error {
186186
return fmt.Errorf("Error adding or updating ingress %v/%v: %v", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err)
187187
}
188188

189-
if err := cnf.nginxManager.Reload(); err != nil {
189+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
190190
return fmt.Errorf("Error reloading NGINX for %v/%v: %v", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err)
191191
}
192192

@@ -222,7 +222,7 @@ func (cnf *Configurator) AddOrUpdateMergeableIngress(mergeableIngs *MergeableIng
222222
return fmt.Errorf("Error when adding or updating ingress %v/%v: %v", mergeableIngs.Master.Ingress.Namespace, mergeableIngs.Master.Ingress.Name, err)
223223
}
224224

225-
if err := cnf.nginxManager.Reload(); err != nil {
225+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
226226
return fmt.Errorf("Error reloading NGINX for %v/%v: %v", mergeableIngs.Master.Ingress.Namespace, mergeableIngs.Master.Ingress.Name, err)
227227
}
228228

@@ -309,7 +309,7 @@ func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServer
309309
return warnings, fmt.Errorf("Error adding or updating VirtualServer %v/%v: %v", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err)
310310
}
311311

312-
if err := cnf.nginxManager.Reload(); err != nil {
312+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
313313
return warnings, fmt.Errorf("Error reloading NGINX for VirtualServer %v/%v: %v", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err)
314314
}
315315

@@ -356,7 +356,7 @@ func (cnf *Configurator) AddOrUpdateVirtualServers(virtualServerExes []*VirtualS
356356
allWarnings.Add(warnings)
357357
}
358358

359-
if err := cnf.nginxManager.Reload(); err != nil {
359+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
360360
return allWarnings, fmt.Errorf("Error when reloading NGINX when updating Policy: %v", err)
361361
}
362362

@@ -371,7 +371,7 @@ func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *Transport
371371
return fmt.Errorf("Error adding or updating TransportServer %v/%v: %v", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err)
372372
}
373373

374-
if err := cnf.nginxManager.Reload(); err != nil {
374+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
375375
return fmt.Errorf("Error reloading NGINX for TransportServer %v/%v: %v", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err)
376376
}
377377

@@ -530,7 +530,7 @@ func (cnf *Configurator) AddOrUpdateTLSSecret(secret *api_v1.Secret, ingExes []I
530530
}
531531
}
532532

533-
if err := cnf.nginxManager.Reload(); err != nil {
533+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
534534
return fmt.Errorf("Error when reloading NGINX when updating Secret: %v", err)
535535
}
536536

@@ -551,7 +551,7 @@ func (cnf *Configurator) AddOrUpdateSpecialTLSSecrets(secret *api_v1.Secret, sec
551551
cnf.nginxManager.CreateSecret(secretName, data, nginx.TLSSecretFileMode)
552552
}
553553

554-
if err := cnf.nginxManager.Reload(); err != nil {
554+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
555555
return fmt.Errorf("Error when reloading NGINX when updating the special Secrets: %v", err)
556556
}
557557

@@ -597,7 +597,7 @@ func (cnf *Configurator) DeleteSecret(key string, ingExes []IngressEx, mergeable
597597
}
598598

599599
if len(ingExes)+len(mergeableIngresses)+len(virtualServerExes) > 0 {
600-
if err := cnf.nginxManager.Reload(); err != nil {
600+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
601601
return fmt.Errorf("Error when reloading NGINX when deleting Secret %v: %v", key, err)
602602
}
603603
}
@@ -617,7 +617,7 @@ func (cnf *Configurator) DeleteIngress(key string) error {
617617
cnf.deleteIngressMetricsLabels(key)
618618
}
619619

620-
if err := cnf.nginxManager.Reload(); err != nil {
620+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
621621
return fmt.Errorf("Error when removing ingress %v: %v", key, err)
622622
}
623623

@@ -634,7 +634,7 @@ func (cnf *Configurator) DeleteVirtualServer(key string) error {
634634
cnf.deleteVirtualServerMetricsLabels(fmt.Sprintf(key))
635635
}
636636

637-
if err := cnf.nginxManager.Reload(); err != nil {
637+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
638638
return fmt.Errorf("Error when removing VirtualServer %v: %v", key, err)
639639
}
640640

@@ -648,7 +648,7 @@ func (cnf *Configurator) DeleteTransportServer(key string) error {
648648
return fmt.Errorf("Error when removing TransportServer %v: %v", key, err)
649649
}
650650

651-
err = cnf.nginxManager.Reload()
651+
err = cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate)
652652
if err != nil {
653653
return fmt.Errorf("Error when removing TransportServer %v: %v", key, err)
654654
}
@@ -694,7 +694,7 @@ func (cnf *Configurator) UpdateEndpoints(ingExes []*IngressEx) error {
694694
return nil
695695
}
696696

697-
if err := cnf.nginxManager.Reload(); err != nil {
697+
if err := cnf.nginxManager.Reload(nginx.ReloadForEndpointsUpdate); err != nil {
698698
return fmt.Errorf("Error reloading NGINX when updating endpoints: %v", err)
699699
}
700700

@@ -727,7 +727,7 @@ func (cnf *Configurator) UpdateEndpointsMergeableIngress(mergeableIngresses []*M
727727
return nil
728728
}
729729

730-
if err := cnf.nginxManager.Reload(); err != nil {
730+
if err := cnf.nginxManager.Reload(nginx.ReloadForEndpointsUpdate); err != nil {
731731
return fmt.Errorf("Error reloading NGINX when updating endpoints for %v: %v", mergeableIngresses, err)
732732
}
733733

@@ -759,7 +759,7 @@ func (cnf *Configurator) UpdateEndpointsForVirtualServers(virtualServerExes []*V
759759
return nil
760760
}
761761

762-
if err := cnf.nginxManager.Reload(); err != nil {
762+
if err := cnf.nginxManager.Reload(nginx.ReloadForEndpointsUpdate); err != nil {
763763
return fmt.Errorf("Error reloading NGINX when updating endpoints: %v", err)
764764
}
765765

@@ -806,7 +806,7 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes
806806
return nil
807807
}
808808

809-
if err := cnf.nginxManager.Reload(); err != nil {
809+
if err := cnf.nginxManager.Reload(nginx.ReloadForEndpointsUpdate); err != nil {
810810
return fmt.Errorf("Error reloading NGINX when updating endpoints: %v", err)
811811
}
812812

@@ -948,7 +948,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, ingExes []*Ingres
948948
}
949949

950950
cnf.nginxManager.SetOpenTracing(mainCfg.OpenTracingLoadModule)
951-
if err := cnf.nginxManager.Reload(); err != nil {
951+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
952952
return allWarnings, fmt.Errorf("Error when updating config from ConfigMap: %v", err)
953953
}
954954

@@ -981,7 +981,7 @@ func (cnf *Configurator) UpdateGlobalConfiguration(globalConfiguration *conf_v1a
981981
}
982982
}
983983

984-
if err := cnf.nginxManager.Reload(); err != nil {
984+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
985985
return updatedTransportServerExes, deletedTransportServerExes, fmt.Errorf("Error when updating global configuration: %v", err)
986986
}
987987

@@ -1097,7 +1097,7 @@ func (cnf *Configurator) AddOrUpdateSpiffeCerts(svidResponse *workload.X509SVIDs
10971097
cnf.nginxManager.CreateSecret(spiffeCertFileName, createSpiffeCert(svid.Certificates), spiffeCertsFileMode)
10981098
cnf.nginxManager.CreateSecret(spiffeBundleFileName, createSpiffeCert(svid.TrustBundle), spiffeCertsFileMode)
10991099

1100-
err = cnf.nginxManager.Reload()
1100+
err = cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate)
11011101
if err != nil {
11021102
return fmt.Errorf("error when reloading NGINX when updating the SPIFFE Certs: %v", err)
11031103
}
@@ -1174,7 +1174,7 @@ func (cnf *Configurator) AddOrUpdateAppProtectResource(resource *unstructured.Un
11741174
}
11751175
}
11761176

1177-
if err := cnf.nginxManager.Reload(); err != nil {
1177+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
11781178
return fmt.Errorf("Error when reloading NGINX when updating %v: %v", resource.GetKind(), err)
11791179
}
11801180

@@ -1201,7 +1201,7 @@ func (cnf *Configurator) DeleteAppProtectPolicy(polNamespaceame string, ingExes
12011201
}
12021202
}
12031203

1204-
if err := cnf.nginxManager.Reload(); err != nil {
1204+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
12051205
return fmt.Errorf("Error when reloading NGINX when removing App Protect Policy: %v", err)
12061206
}
12071207

@@ -1228,7 +1228,7 @@ func (cnf *Configurator) DeleteAppProtectLogConf(logConfNamespaceame string, ing
12281228
}
12291229
}
12301230

1231-
if err := cnf.nginxManager.Reload(); err != nil {
1231+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
12321232
return fmt.Errorf("Error when reloading NGINX when removing App Protect Log Configuration: %v", err)
12331233
}
12341234

@@ -1246,7 +1246,7 @@ func (cnf *Configurator) AddInternalRouteConfig() error {
12461246
return fmt.Errorf("Error when writing main Config: %v", err)
12471247
}
12481248
cnf.nginxManager.CreateMainConfig(mainCfgContent)
1249-
if err := cnf.nginxManager.Reload(); err != nil {
1249+
if err := cnf.nginxManager.Reload(nginx.ReloadForOtherUpdate); err != nil {
12501250
return fmt.Errorf("Error when reloading nginx: %v", err)
12511251
}
12521252
return nil

internal/metrics/collectors/manager.go

+15-6
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88

99
// ManagerCollector is an interface for the metrics of the Nginx Manager
1010
type ManagerCollector interface {
11-
IncNginxReloadCount()
11+
IncNginxReloadCount(isEndPointUpdate bool)
1212
IncNginxReloadErrors()
1313
UpdateLastReloadTime(ms time.Duration)
1414
Register(registry *prometheus.Registry) error
@@ -17,7 +17,7 @@ type ManagerCollector interface {
1717
// LocalManagerMetricsCollector implements NginxManagerCollector interface and prometheus.Collector interface
1818
type LocalManagerMetricsCollector struct {
1919
// Metrics
20-
reloadsTotal prometheus.Counter
20+
reloadsTotal *prometheus.CounterVec
2121
reloadsError prometheus.Counter
2222
lastReloadStatus prometheus.Gauge
2323
lastReloadTime prometheus.Gauge
@@ -26,13 +26,14 @@ type LocalManagerMetricsCollector struct {
2626
// NewLocalManagerMetricsCollector creates a new LocalManagerMetricsCollector
2727
func NewLocalManagerMetricsCollector(constLabels map[string]string) *LocalManagerMetricsCollector {
2828
nc := &LocalManagerMetricsCollector{
29-
reloadsTotal: prometheus.NewCounter(
29+
reloadsTotal: prometheus.NewCounterVec(
3030
prometheus.CounterOpts{
3131
Name: "nginx_reloads_total",
3232
Namespace: metricsNamespace,
3333
Help: "Number of successful NGINX reloads",
3434
ConstLabels: constLabels,
3535
},
36+
[]string{"reason"},
3637
),
3738
reloadsError: prometheus.NewCounter(
3839
prometheus.CounterOpts{
@@ -59,12 +60,20 @@ func NewLocalManagerMetricsCollector(constLabels map[string]string) *LocalManage
5960
},
6061
),
6162
}
63+
nc.reloadsTotal.WithLabelValues("other")
64+
nc.reloadsTotal.WithLabelValues("endpoints")
6265
return nc
6366
}
6467

6568
// IncNginxReloadCount increments the counter of successful NGINX reloads and sets the last reload status to true
66-
func (nc *LocalManagerMetricsCollector) IncNginxReloadCount() {
67-
nc.reloadsTotal.Inc()
69+
func (nc *LocalManagerMetricsCollector) IncNginxReloadCount(isEndPointUpdate bool) {
70+
var label string
71+
if isEndPointUpdate {
72+
label = "endpoints"
73+
} else {
74+
label = "other"
75+
}
76+
nc.reloadsTotal.WithLabelValues(label).Inc()
6877
nc.updateLastReloadStatus(true)
6978
}
7079

@@ -121,7 +130,7 @@ func NewManagerFakeCollector() *ManagerFakeCollector {
121130
func (nc *ManagerFakeCollector) Register(registry *prometheus.Registry) error { return nil }
122131

123132
// IncNginxReloadCount implements a fake IncNginxReloadCount
124-
func (nc *ManagerFakeCollector) IncNginxReloadCount() {}
133+
func (nc *ManagerFakeCollector) IncNginxReloadCount(isEndPointUpdate bool) {}
125134

126135
// IncNginxReloadErrors implements a fake IncNginxReloadErrors
127136
func (nc *ManagerFakeCollector) IncNginxReloadErrors() {}

internal/nginx/fake_manager.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func (*FakeManager) Start(done chan error) {
9797
}
9898

9999
// Reload provides a fake implementation of Reload.
100-
func (*FakeManager) Reload() error {
100+
func (*FakeManager) Reload(isEndpointsUpdate bool) error {
101101
glog.V(3).Infof("Reloading nginx")
102102
return nil
103103
}

internal/nginx/manager.go

+9-3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ import (
1515
"github.com/nginxinc/nginx-plus-go-client/client"
1616
)
1717

18+
// ReloadForEndpointsUpdate means that is caused by an endpoints update.
19+
const ReloadForEndpointsUpdate = true
20+
21+
// ReloadForOtherUpdate means that a reload is caused by an update for a resource(s) other than endpoints.
22+
const ReloadForOtherUpdate = false
23+
1824
// TLSSecretFileMode defines the default filemode for files with TLS Secrets.
1925
const TLSSecretFileMode = 0600
2026

@@ -61,7 +67,7 @@ type Manager interface {
6167
CreateDHParam(content string) (string, error)
6268
CreateOpenTracingTracerConfig(content string) error
6369
Start(done chan error)
64-
Reload() error
70+
Reload(isEndpointsUpdate bool) error
6571
Quit()
6672
UpdateConfigVersionFile(openTracing bool)
6773
SetPlusClients(plusClient *client.NginxClient, plusConfigVersionCheckClient *http.Client)
@@ -268,7 +274,7 @@ func (lm *LocalManager) Start(done chan error) {
268274
}
269275

270276
// Reload reloads NGINX.
271-
func (lm *LocalManager) Reload() error {
277+
func (lm *LocalManager) Reload(isEndpointsUpdate bool) error {
272278
// write a new config version
273279
lm.configVersion++
274280
lm.UpdateConfigVersionFile(lm.OpenTracing)
@@ -287,7 +293,7 @@ func (lm *LocalManager) Reload() error {
287293
return fmt.Errorf("could not get newest config version: %v", err)
288294
}
289295

290-
lm.metricsCollector.IncNginxReloadCount()
296+
lm.metricsCollector.IncNginxReloadCount(isEndpointsUpdate)
291297

292298
t2 := time.Now()
293299
lm.metricsCollector.UpdateLastReloadTime(t2.Sub(t1))

0 commit comments

Comments
 (0)