Skip to content

Commit f9381e0

Browse files
benluddydinhxuanvuawgreene
committed
Use one HTTP server for healthz, metrics, and profiling.
TLS can still be enabled via flags, but it now applies to all three APIs since there is a single HTTP server per operator process. The -profiling flag is deprecated and does nothing. The profiling APIs are now always enabled, but they refuse to serve clients that do not present a verifiable certificate (see the new flag -client-ca). This effectively disables the profiling APIs when served over HTTP. Co-authored-by: Vu Dinh <[email protected]> Co-authored-by: Alexander Greene <[email protected]> Signed-off-by: Ben Luddy <[email protected]>
1 parent 4ffa5c1 commit f9381e0

14 files changed

+295
-192
lines changed

‎cmd/catalog/main.go

+11-56
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,23 @@ package main
22

33
import (
44
"context"
5-
"crypto/tls"
65
"flag"
76
"fmt"
87
"net/http"
98
"os"
109
"time"
1110

1211
configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
13-
"github.com/prometheus/client_golang/prometheus/promhttp"
1412
log "github.com/sirupsen/logrus"
1513
utilclock "k8s.io/apimachinery/pkg/util/clock"
1614
k8sscheme "k8s.io/client-go/kubernetes/scheme"
1715
"k8s.io/client-go/tools/clientcmd"
1816

1917
"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
2018
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/catalog"
21-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor"
2219
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
2320
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
24-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
21+
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server"
2522
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
2623
"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
2724
olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
@@ -67,8 +64,9 @@ var (
6764
tlsCertPath = flag.String(
6865
"tls-cert", "", "Path to use for certificate key (requires tls-key)")
6966

70-
profiling = flag.Bool(
71-
"profiling", false, "serve profiling data (on port 8080)")
67+
profiling = flag.Bool("profiling", false, "deprecated")
68+
69+
clientCAPath = flag.String("client-ca", "", "path to watch for client ca bundle")
7270

7371
installPlanTimeout = flag.Duration("install-plan-retry-timeout", 1*time.Minute, "time since first attempt at which plan execution errors are considered fatal")
7472
bundleUnpackTimeout = flag.Duration("bundle-unpack-timeout", 10*time.Minute, "The time limit for bundle unpacking, after which InstallPlan execution is considered to have failed. 0 is considered as having no timeout.")
@@ -106,59 +104,16 @@ func main() {
106104
*catalogNamespace = catalogNamespaceEnvVarValue
107105
}
108106

109-
var useTLS bool
110-
if *tlsCertPath != "" && *tlsKeyPath == "" || *tlsCertPath == "" && *tlsKeyPath != "" {
111-
logger.Warn("both --tls-key and --tls-crt must be provided for TLS to be enabled, falling back to non-https")
112-
} else if *tlsCertPath == "" && *tlsKeyPath == "" {
113-
logger.Info("TLS keys not set, using non-https for metrics")
114-
} else {
115-
logger.Info("TLS keys set, using https for metrics")
116-
useTLS = true
117-
}
118-
119-
// Serve a health check.
120-
healthMux := http.NewServeMux()
121-
healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
122-
w.WriteHeader(http.StatusOK)
123-
})
124-
125-
// Serve profiling if enabled
126-
if *profiling {
127-
logger.Infof("profiling enabled")
128-
profile.RegisterHandlers(healthMux)
107+
listenAndServe, err := server.GetListenAndServeFunc(logger, tlsCertPath, tlsKeyPath, clientCAPath)
108+
if err != nil {
109+
logger.Fatal("Error setting up health/metric/pprof service: %v", err)
129110
}
130111

131-
go http.ListenAndServe(":8080", healthMux)
132-
133-
metricsMux := http.NewServeMux()
134-
metricsMux.Handle("/metrics", promhttp.Handler())
135-
if useTLS {
136-
tlsGetCertFn, err := filemonitor.OLMGetCertRotationFn(logger, *tlsCertPath, *tlsKeyPath)
137-
if err != nil {
138-
logger.Errorf("Certificate monitoring for metrics (https) failed: %v", err)
112+
go func() {
113+
if err := listenAndServe(); err != nil && err != http.ErrServerClosed {
114+
logger.Error(err)
139115
}
140-
141-
go func() {
142-
httpsServer := &http.Server{
143-
Addr: ":8081",
144-
Handler: metricsMux,
145-
TLSConfig: &tls.Config{
146-
GetCertificate: tlsGetCertFn,
147-
},
148-
}
149-
err := httpsServer.ListenAndServeTLS("", "")
150-
if err != nil {
151-
logger.Errorf("Metrics (https) serving failed: %v", err)
152-
}
153-
}()
154-
} else {
155-
go func() {
156-
err := http.ListenAndServe(":8081", metricsMux)
157-
if err != nil {
158-
logger.Errorf("Metrics (http) serving failed: %v", err)
159-
}
160-
}()
161-
}
116+
}()
162117

163118
// create a config client for operator status
164119
config, err := clientcmd.BuildConfigFromFlags("", *kubeConfigPath)

‎cmd/olm/main.go

+9-59
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package main
22

33
import (
44
"context"
5-
"crypto/tls"
65
"flag"
76
"fmt"
87
"net/http"
@@ -12,7 +11,6 @@ import (
1211

1312
configclientset "github.com/openshift/client-go/config/clientset/versioned"
1413
configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
15-
"github.com/prometheus/client_golang/prometheus/promhttp"
1614
"github.com/sirupsen/logrus"
1715
"github.com/spf13/pflag"
1816
v1 "k8s.io/api/core/v1"
@@ -23,11 +21,10 @@ import (
2321
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm"
2422
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/openshift"
2523
"github.com/operator-framework/operator-lifecycle-manager/pkg/feature"
26-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor"
2724
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
2825
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
29-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
3026
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer"
27+
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server"
3128
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
3229
"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
3330
olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
@@ -66,8 +63,9 @@ var (
6663
tlsCertPath = pflag.String(
6764
"tls-cert", "", "Path to use for certificate key (requires tls-key)")
6865

69-
profiling = pflag.Bool(
70-
"profiling", false, "serve profiling data (on port 8080)")
66+
profiling = pflag.Bool("profiling", false, "deprecated")
67+
68+
clientCAPath = pflag.String("client-ca", "", "path to watch for client ca bundle")
7169

7270
namespace = pflag.String(
7371
"namespace", "", "namespace where cleanup runs")
@@ -120,65 +118,17 @@ func main() {
120118
}
121119
logger.Infof("log level %s", logger.Level)
122120

123-
var useTLS bool
124-
if *tlsCertPath != "" && *tlsKeyPath == "" || *tlsCertPath == "" && *tlsKeyPath != "" {
125-
logger.Warn("both --tls-key and --tls-crt must be provided for TLS to be enabled, falling back to non-https")
126-
} else if *tlsCertPath == "" && *tlsKeyPath == "" {
127-
logger.Info("TLS keys not set, using non-https for metrics")
128-
} else {
129-
logger.Info("TLS keys set, using https for metrics")
130-
useTLS = true
131-
}
132-
133-
// Serve a health check.
134-
healthMux := http.NewServeMux()
135-
healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
136-
w.WriteHeader(http.StatusOK)
137-
})
138-
139-
// Serve profiling if enabled
140-
if *profiling {
141-
logger.Infof("profiling enabled")
142-
profile.RegisterHandlers(healthMux)
121+
listenAndServe, err := server.GetListenAndServeFunc(logger, tlsCertPath, tlsKeyPath, clientCAPath)
122+
if err != nil {
123+
logger.Fatal("Error setting up health/metric/pprof service: %v", err)
143124
}
144125

145126
go func() {
146-
err := http.ListenAndServe(":8080", healthMux)
147-
if err != nil {
148-
logger.Errorf("Health serving failed: %v", err)
127+
if err := listenAndServe(); err != nil && err != http.ErrServerClosed {
128+
logger.Error(err)
149129
}
150130
}()
151131

152-
metricsMux := http.NewServeMux()
153-
metricsMux.Handle("/metrics", promhttp.Handler())
154-
if useTLS {
155-
tlsGetCertFn, err := filemonitor.OLMGetCertRotationFn(logger, *tlsCertPath, *tlsKeyPath)
156-
if err != nil {
157-
logger.Errorf("Certificate monitoring for metrics (https) failed: %v", err)
158-
}
159-
160-
go func() {
161-
httpsServer := &http.Server{
162-
Addr: ":8081",
163-
Handler: metricsMux,
164-
TLSConfig: &tls.Config{
165-
GetCertificate: tlsGetCertFn,
166-
},
167-
}
168-
err := httpsServer.ListenAndServeTLS("", "")
169-
if err != nil {
170-
logger.Errorf("Metrics (https) serving failed: %v", err)
171-
}
172-
}()
173-
} else {
174-
go func() {
175-
err := http.ListenAndServe(":8081", metricsMux)
176-
if err != nil {
177-
logger.Errorf("Metrics (http) serving failed: %v", err)
178-
}
179-
}()
180-
}
181-
182132
mgr, err := Manager(ctx, *debug)
183133
if err != nil {
184134
logger.WithError(err).Fatalf("error configuring controller manager")

‎deploy/chart/templates/0000_50_olm_02-services.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ spec:
1212
type: ClusterIP
1313
ports:
1414
- name: https-metrics
15-
port: 8081
15+
port: {{ .Values.olm.service.externalPort }}
1616
protocol: TCP
17-
targetPort: metrics
17+
targetPort: {{ .Values.olm.service.internalPort }}
1818
selector:
1919
app: olm-operator
2020
---
@@ -31,9 +31,9 @@ spec:
3131
type: ClusterIP
3232
ports:
3333
- name: https-metrics
34-
port: 8081
34+
port: {{ .Values.catalog.service.externalPort }}
3535
protocol: TCP
36-
targetPort: metrics
36+
targetPort: {{ .Values.catalog.service.internalPort }}
3737
selector:
3838
app: catalog-operator
3939
{{ end }}

‎deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml

+35-8
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,34 @@ spec:
1818
app: olm-operator
1919
spec:
2020
serviceAccountName: olm-operator-serviceaccount
21+
{{- if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}
22+
volumes:
23+
{{- end }}
24+
{{- if .Values.olm.tlsSecret }}
25+
- name: srv-cert
26+
secret:
27+
secretName: {{ .Values.olm.tlsSecret }}
28+
{{- end }}
29+
{{- if .Values.olm.clientCASecret }}
30+
- name: profile-collector-cert
31+
secret:
32+
secretName: {{ .Values.olm.clientCASecret }}
33+
{{- end }}
2134
containers:
2235
- name: olm-operator
36+
{{- if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}
37+
volumeMounts:
38+
{{- end }}
39+
{{- if .Values.olm.tlsSecret }}
40+
- name: srv-cert
41+
mountPath: "/srv-cert"
42+
readOnly: true
43+
{{- end }}
44+
{{- if .Values.olm.clientCASecret }}
45+
- name: profile-collector-cert
46+
mountPath: "/profile-collector-cert"
47+
readOnly: true
48+
{{- end }}
2349
command:
2450
- /bin/olm
2551
args:
@@ -43,29 +69,30 @@ spec:
4369
- --writePackageServerStatusName
4470
- {{ .Values.writePackageServerStatusName }}
4571
{{- end }}
46-
{{- if .Values.olm.tlsCertPath }}
72+
{{- if .Values.olm.tlsSecret }}
4773
- --tls-cert
48-
- {{ .Values.olm.tlsCertPath }}
49-
{{- end }}
50-
{{- if .Values.olm.tlsKeyPath }}
74+
- /srv-cert/tls.crt
5175
- --tls-key
52-
- {{ .Values.olm.tlsKeyPath }}
76+
- /srv-cert/tls.key
77+
{{- end }}
78+
{{- if .Values.olm.clientCASecret }}
79+
- --client-ca
80+
- /profile-collector-cert/tls.crt
5381
{{- end }}
5482
image: {{ .Values.olm.image.ref }}
5583
imagePullPolicy: {{ .Values.olm.image.pullPolicy }}
5684
ports:
5785
- containerPort: {{ .Values.olm.service.internalPort }}
58-
- containerPort: 8081
59-
name: metrics
60-
protocol: TCP
6186
livenessProbe:
6287
httpGet:
6388
path: /healthz
6489
port: {{ .Values.olm.service.internalPort }}
90+
scheme: {{ if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}HTTPS{{ else }}HTTP{{end}}
6591
readinessProbe:
6692
httpGet:
6793
path: /healthz
6894
port: {{ .Values.olm.service.internalPort }}
95+
scheme: {{ if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}HTTPS{{ else }}HTTP{{end}}
6996
terminationMessagePolicy: FallbackToLogsOnError
7097
env:
7198
- name: OPERATOR_NAMESPACE

‎deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml

+36-9
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,34 @@ spec:
1818
app: catalog-operator
1919
spec:
2020
serviceAccountName: olm-operator-serviceaccount
21+
{{- if or .Values.catalog.tlsSecret .Values.catalog.clientCASecret }}
22+
volumes:
23+
{{- end }}
24+
{{- if .Values.catalog.tlsSecret }}
25+
- name: srv-cert
26+
secret:
27+
secretName: {{ .Values.catalog.tlsSecret }}
28+
{{- end }}
29+
{{- if .Values.catalog.clientCASecret }}
30+
- name: profile-collector-cert
31+
secret:
32+
secretName: {{ .Values.catalog.clientCASecret }}
33+
{{- end }}
2134
containers:
2235
- name: catalog-operator
36+
{{- if or .Values.catalog.tlsSecret .Values.catalog.clientCASecret }}
37+
volumeMounts:
38+
{{- end }}
39+
{{- if .Values.catalog.tlsSecret }}
40+
- name: srv-cert
41+
mountPath: "/srv-cert"
42+
readOnly: true
43+
{{- end }}
44+
{{- if .Values.catalog.clientCASecret }}
45+
- name: profile-collector-cert
46+
mountPath: "/profile-collector-cert"
47+
readOnly: true
48+
{{- end }}
2349
command:
2450
- /bin/catalog
2551
args:
@@ -37,29 +63,30 @@ spec:
3763
- -writeStatusName
3864
- {{ .Values.writeStatusNameCatalog }}
3965
{{- end }}
40-
{{- if .Values.olm.tlsCertPath }}
41-
- -tls-cert
42-
- {{ .Values.olm.tlsCertPath }}
66+
{{- if .Values.catalog.tlsSecret }}
67+
- --tls-cert
68+
- /srv-cert/tls.crt
69+
- --tls-key
70+
- /srv-cert/tls.key
4371
{{- end }}
44-
{{- if .Values.olm.tlsKeyPath }}
45-
- -tls-key
46-
- {{ .Values.olm.tlsKeyPath }}
72+
{{- if .Values.catalog.clientCASecret }}
73+
- --client-ca
74+
- /profile-collector-cert/tls.crt
4775
{{- end }}
4876
image: {{ .Values.catalog.image.ref }}
4977
imagePullPolicy: {{ .Values.catalog.image.pullPolicy }}
5078
ports:
5179
- containerPort: {{ .Values.catalog.service.internalPort }}
52-
- containerPort: 8081
53-
name: metrics
54-
protocol: TCP
5580
livenessProbe:
5681
httpGet:
5782
path: /healthz
5883
port: {{ .Values.catalog.service.internalPort }}
84+
scheme: {{ if and .Values.catalog.tlsKeyPath .Values.catalog.tlsCertPath }}HTTPS{{ else }}HTTP{{end}}
5985
readinessProbe:
6086
httpGet:
6187
path: /healthz
6288
port: {{ .Values.catalog.service.internalPort }}
89+
scheme: {{ if and .Values.catalog.tlsKeyPath .Values.catalog.tlsCertPath }}HTTPS{{ else }}HTTP{{end}}
6390
terminationMessagePolicy: FallbackToLogsOnError
6491
{{- if .Values.catalog.resources }}
6592
resources:

0 commit comments

Comments
 (0)