Skip to content

Commit a0424dd

Browse files
smarterclaytonsmarterclayton
committed
Add a test suite that verifies the router metrics
Tests the metrics endpoint, including metrics transformation, healthz, and ACL checks.
1 parent 20a8531 commit a0424dd

File tree

4 files changed

+356
-0
lines changed

4 files changed

+356
-0
lines changed

‎test/extended/router/metrics.go

+245
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
package images
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"net/http"
7+
"strconv"
8+
"time"
9+
10+
g "github.com/onsi/ginkgo"
11+
o "github.com/onsi/gomega"
12+
dto "github.com/prometheus/client_model/go"
13+
"github.com/prometheus/common/expfmt"
14+
15+
kapi "k8s.io/kubernetes/pkg/api"
16+
kapierrs "k8s.io/kubernetes/pkg/api/errors"
17+
"k8s.io/kubernetes/pkg/util/sets"
18+
e2e "k8s.io/kubernetes/test/e2e/framework"
19+
20+
exutil "github.com/openshift/origin/test/extended/util"
21+
)
22+
23+
var _ = g.Describe("[Conformance][networking][router] openshift router metrics", func() {
24+
defer g.GinkgoRecover()
25+
var (
26+
oc = exutil.NewCLI("router-metrics", exutil.KubeConfigPath())
27+
28+
username, password, execPodName, ns, host string
29+
)
30+
31+
g.BeforeEach(func() {
32+
dc, err := oc.AdminClient().DeploymentConfigs("default").Get("router")
33+
if kapierrs.IsNotFound(err) {
34+
g.Skip("no router installed on the cluster")
35+
return
36+
}
37+
o.Expect(err).NotTo(o.HaveOccurred())
38+
env := dc.Spec.Template.Spec.Containers[0].Env
39+
username, password = findEnvVar(env, "STATS_USERNAME"), findEnvVar(env, "STATS_PASSWORD")
40+
41+
epts, err := oc.AdminKubeClient().Endpoints("default").Get("router")
42+
o.Expect(err).NotTo(o.HaveOccurred())
43+
host = epts.Subsets[0].Addresses[0].IP
44+
45+
ns = oc.KubeFramework().Namespace.Name
46+
})
47+
48+
g.Describe("The HAProxy router", func() {
49+
g.It("should expose a health check on the metrics port", func() {
50+
execPodName = exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
51+
defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, kapi.NewDeleteOptions(1)) }()
52+
53+
g.By("listening on the health port")
54+
err := expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:1935/healthz", host), 200)
55+
o.Expect(err).NotTo(o.HaveOccurred())
56+
})
57+
58+
g.It("should expose prometheus metrics for a route", func() {
59+
g.By("when a route exists")
60+
configPath := exutil.FixturePath("testdata", "router-metrics.yaml")
61+
err := oc.Run("create").Args("-f", configPath).Execute()
62+
o.Expect(err).NotTo(o.HaveOccurred())
63+
64+
execPodName = exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
65+
defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, kapi.NewDeleteOptions(1)) }()
66+
67+
g.By("preventing access without a username and password")
68+
err = expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:1935/metrics", host), 403)
69+
o.Expect(err).NotTo(o.HaveOccurred())
70+
71+
g.By("checking for the expected metrics")
72+
routeLabels := labels{"backend": "http", "namespace": ns, "route": "weightedroute"}
73+
var metrics map[string]*dto.MetricFamily
74+
times := 10
75+
var results string
76+
defer func() { e2e.Logf("received metrics:\n%s", results) }()
77+
for {
78+
results, err = getAuthenticatedURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:1935/metrics", host), username, password)
79+
o.Expect(err).NotTo(o.HaveOccurred())
80+
81+
p := expfmt.TextParser{}
82+
metrics, err = p.TextToMetricFamilies(bytes.NewBufferString(results))
83+
o.Expect(err).NotTo(o.HaveOccurred())
84+
85+
if len(findGaugesWithLabels(metrics["haproxy_server_up"], routeLabels)) == 2 {
86+
if findGaugesWithLabels(metrics["haproxy_server_connections_total"], routeLabels)[0] > 0 {
87+
break
88+
}
89+
// send a burst of traffic to the router
90+
g.By("sending traffic to a weighted route")
91+
err = expectRouteStatusCodeRepeatedExec(ns, execPodName, fmt.Sprintf("http://%s", host), "weighted.example.com", http.StatusOK, times)
92+
o.Expect(err).NotTo(o.HaveOccurred())
93+
}
94+
time.Sleep(2 * time.Second)
95+
g.By("retrying metrics until all backend servers appear")
96+
}
97+
98+
allEndpoints := sets.NewString()
99+
for _, name := range []string{"weightedendpoints1", "weightedendpoints2"} {
100+
epts, err := oc.AdminKubeClient().Endpoints(ns).Get(name)
101+
o.Expect(err).NotTo(o.HaveOccurred())
102+
for _, s := range epts.Subsets {
103+
for _, a := range s.Addresses {
104+
allEndpoints.Insert(a.IP + ":8080")
105+
}
106+
}
107+
}
108+
foundEndpoints := sets.NewString(findMetricLabels(metrics["haproxy_server_http_responses_total"], routeLabels, "server")...)
109+
o.Expect(allEndpoints.List()).To(o.Equal(foundEndpoints.List()))
110+
111+
// route specific metrics from server and backend
112+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_http_responses_total"], routeLabels.With("code", "2xx"))).To(o.ConsistOf(o.BeNumerically(">", 0), o.BeNumerically(">", 0)))
113+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_http_responses_total"], routeLabels.With("code", "5xx"))).To(o.Equal([]float64{0, 0}))
114+
// only server returns response counts
115+
o.Expect(findGaugesWithLabels(metrics["haproxy_backend_http_responses_total"], routeLabels.With("code", "2xx"))).To(o.HaveLen(0))
116+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_connections_total"], routeLabels)).To(o.ConsistOf(o.BeNumerically(">=", 0), o.BeNumerically(">=", 0)))
117+
o.Expect(findGaugesWithLabels(metrics["haproxy_backend_connections_total"], routeLabels)).To(o.ConsistOf(o.BeNumerically(">=", times)))
118+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_up"], routeLabels)).To(o.Equal([]float64{1, 1}))
119+
o.Expect(findGaugesWithLabels(metrics["haproxy_backend_up"], routeLabels)).To(o.Equal([]float64{1}))
120+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_bytes_in_total"], routeLabels)).To(o.ConsistOf(o.BeNumerically(">=", 0), o.BeNumerically(">=", 0)))
121+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_bytes_out_total"], routeLabels)).To(o.ConsistOf(o.BeNumerically(">=", 0), o.BeNumerically(">=", 0)))
122+
o.Expect(findGaugesWithLabels(metrics["haproxy_server_max_sessions"], routeLabels)).To(o.ConsistOf(o.BeNumerically(">", 0), o.BeNumerically(">", 0)))
123+
124+
// generic metrics
125+
o.Expect(findGaugesWithLabels(metrics["haproxy_up"], nil)).To(o.Equal([]float64{1}))
126+
o.Expect(findGaugesWithLabels(metrics["haproxy_exporter_scrape_interval"], nil)).To(o.ConsistOf(o.BeNumerically(">", 0)))
127+
o.Expect(findCountersWithLabels(metrics["haproxy_exporter_total_scrapes"], nil)).To(o.ConsistOf(o.BeNumerically(">", 0)))
128+
o.Expect(findCountersWithLabels(metrics["haproxy_exporter_csv_parse_failures"], nil)).To(o.Equal([]float64{0}))
129+
o.Expect(findGaugesWithLabels(metrics["haproxy_process_resident_memory_bytes"], nil)).To(o.ConsistOf(o.BeNumerically(">", 0)))
130+
o.Expect(findGaugesWithLabels(metrics["haproxy_process_max_fds"], nil)).To(o.ConsistOf(o.BeNumerically(">", 0)))
131+
o.Expect(findGaugesWithLabels(metrics["openshift_build_info"], nil)).To(o.Equal([]float64{1}))
132+
133+
// router metrics
134+
o.Expect(findMetricsWithLabels(metrics["template_router_reload_seconds"], nil)[0].Summary.GetSampleSum()).To(o.BeNumerically(">", 0))
135+
o.Expect(findMetricsWithLabels(metrics["template_router_write_config_seconds"], nil)[0].Summary.GetSampleSum()).To(o.BeNumerically(">", 0))
136+
})
137+
138+
g.It("should expose the profiling endpoints", func() {
139+
execPodName = exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
140+
defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, kapi.NewDeleteOptions(1)) }()
141+
142+
g.By("preventing access without a username and password")
143+
err := expectURLStatusCodeExec(ns, execPodName, fmt.Sprintf("http://%s:1935/debug/pprof/heap", host), 403)
144+
o.Expect(err).NotTo(o.HaveOccurred())
145+
146+
g.By("at /debug/pprof")
147+
results, err := getAuthenticatedURLViaPod(ns, execPodName, fmt.Sprintf("http://%s:1935/debug/pprof/heap", host), username, password)
148+
o.Expect(err).NotTo(o.HaveOccurred())
149+
o.Expect(results).To(o.ContainSubstring("# runtime.MemStats"))
150+
})
151+
})
152+
})
153+
154+
type labels map[string]string
155+
156+
func (l labels) With(name, value string) labels {
157+
n := make(labels)
158+
for k, v := range l {
159+
n[k] = v
160+
}
161+
n[name] = value
162+
return n
163+
}
164+
165+
func findEnvVar(vars []kapi.EnvVar, key string) string {
166+
for _, v := range vars {
167+
if v.Name == key {
168+
return v.Value
169+
}
170+
}
171+
return ""
172+
}
173+
174+
func findMetricsWithLabels(f *dto.MetricFamily, labels map[string]string) []*dto.Metric {
175+
var result []*dto.Metric
176+
if f == nil {
177+
return result
178+
}
179+
for _, m := range f.Metric {
180+
matched := map[string]struct{}{}
181+
for _, l := range m.Label {
182+
if expect, ok := labels[l.GetName()]; ok {
183+
if expect != l.GetValue() {
184+
break
185+
}
186+
matched[l.GetName()] = struct{}{}
187+
}
188+
}
189+
if len(matched) != len(labels) {
190+
continue
191+
}
192+
result = append(result, m)
193+
}
194+
return result
195+
}
196+
197+
func findCountersWithLabels(f *dto.MetricFamily, labels map[string]string) []float64 {
198+
var result []float64
199+
for _, m := range findMetricsWithLabels(f, labels) {
200+
result = append(result, m.Counter.GetValue())
201+
}
202+
return result
203+
}
204+
205+
func findGaugesWithLabels(f *dto.MetricFamily, labels map[string]string) []float64 {
206+
var result []float64
207+
for _, m := range findMetricsWithLabels(f, labels) {
208+
result = append(result, m.Gauge.GetValue())
209+
}
210+
return result
211+
}
212+
213+
func findMetricLabels(f *dto.MetricFamily, labels map[string]string, match string) []string {
214+
var result []string
215+
for _, m := range findMetricsWithLabels(f, labels) {
216+
for _, l := range m.Label {
217+
if l.GetName() == match {
218+
result = append(result, l.GetValue())
219+
break
220+
}
221+
}
222+
}
223+
return result
224+
}
225+
226+
func expectURLStatusCodeExec(ns, execPodName, url string, statusCode int) error {
227+
cmd := fmt.Sprintf("curl -s -o /dev/null -w '%%{http_code}' %q", url)
228+
output, err := e2e.RunHostCmd(ns, execPodName, cmd)
229+
if err != nil {
230+
return fmt.Errorf("host command failed: %v\n%s", err, output)
231+
}
232+
if output != strconv.Itoa(statusCode) {
233+
return fmt.Errorf("last response from server was not %d: %s", statusCode, output)
234+
}
235+
return nil
236+
}
237+
238+
func getAuthenticatedURLViaPod(ns, execPodName, url, user, pass string) (string, error) {
239+
cmd := fmt.Sprintf("curl -s -u %s:%s %q", user, pass, url)
240+
output, err := e2e.RunHostCmd(ns, execPodName, cmd)
241+
if err != nil {
242+
return "", fmt.Errorf("host command failed: %v\n%s", err, output)
243+
}
244+
return output, nil
245+
}

‎test/extended/testdata/router-metrics.yaml

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
apiVersion: v1
2+
kind: List
3+
items:
4+
# a route that has multiple weighted services that it points to
5+
- apiVersion: v1
6+
kind: Route
7+
metadata:
8+
name: weightedroute
9+
labels:
10+
test: router
11+
select: weighted
12+
spec:
13+
host: weighted.example.com
14+
to:
15+
name: weightedendpoints1
16+
kind: Service
17+
weight: 50
18+
alternateBackends:
19+
- name: weightedendpoints2
20+
kind: Service
21+
weight: 50
22+
ports:
23+
- targetPort: 8080
24+
25+
# a route that has multiple services but all weights are zero
26+
- apiVersion: v1
27+
kind: Route
28+
metadata:
29+
name: zeroweightroute
30+
labels:
31+
test: router
32+
select: weighted
33+
spec:
34+
host: zeroweight.example.com
35+
to:
36+
name: weightedendpoints1
37+
kind: Service
38+
weight: 0
39+
alternateBackends:
40+
- name: weightedendpoints2
41+
kind: Service
42+
weight: 0
43+
ports:
44+
- targetPort: 8080
45+
46+
# two services that can be routed to
47+
- apiVersion: v1
48+
kind: Service
49+
metadata:
50+
name: weightedendpoints1
51+
labels:
52+
test: router
53+
spec:
54+
selector:
55+
test: weightedrouter1
56+
endpoints: weightedrouter1
57+
ports:
58+
- port: 8080
59+
- apiVersion: v1
60+
kind: Service
61+
metadata:
62+
name: weightedendpoints2
63+
labels:
64+
test: router
65+
spec:
66+
selector:
67+
test: weightedrouter2
68+
endpoints: weightedrouter2
69+
ports:
70+
- port: 8080
71+
# two pods that serves a response
72+
- apiVersion: v1
73+
kind: Pod
74+
metadata:
75+
name: endpoint-1
76+
labels:
77+
test: weightedrouter1
78+
endpoints: weightedrouter1
79+
spec:
80+
terminationGracePeriodSeconds: 1
81+
containers:
82+
- name: test
83+
image: openshift/hello-openshift
84+
ports:
85+
- containerPort: 8080
86+
name: http
87+
- containerPort: 100
88+
protocol: UDP
89+
- apiVersion: v1
90+
kind: Pod
91+
metadata:
92+
name: endpoint-2
93+
labels:
94+
test: weightedrouter2
95+
endpoints: weightedrouter2
96+
spec:
97+
terminationGracePeriodSeconds: 1
98+
containers:
99+
- name: test
100+
image: openshift/hello-openshift
101+
ports:
102+
- containerPort: 8080
103+
name: http
104+
- containerPort: 100
105+
protocol: UDP

‎test/extended/testdata/scoped-router.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ items:
99
labels:
1010
test: scoped-router
1111
spec:
12+
terminationGracePeriodSeconds: 1
1213
containers:
1314
- name: router
1415
image: openshift/origin-haproxy-router
@@ -35,6 +36,7 @@ items:
3536
labels:
3637
test: router-override
3738
spec:
39+
terminationGracePeriodSeconds: 1
3840
containers:
3941
- name: router
4042
image: openshift/origin-haproxy-router
@@ -115,6 +117,7 @@ items:
115117
test: router
116118
endpoints: router
117119
spec:
120+
terminationGracePeriodSeconds: 1
118121
containers:
119122
- name: test
120123
image: openshift/hello-openshift

‎test/extended/testdata/weighted-router.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ items:
99
labels:
1010
test: weighted-router
1111
spec:
12+
terminationGracePeriodSeconds: 1
1213
containers:
1314
- name: router
1415
image: openshift/origin-haproxy-router
@@ -114,6 +115,7 @@ items:
114115
test: weightedrouter1
115116
endpoints: weightedrouter1
116117
spec:
118+
terminationGracePeriodSeconds: 1
117119
containers:
118120
- name: test
119121
image: openshift/hello-openshift
@@ -130,6 +132,7 @@ items:
130132
test: weightedrouter2
131133
endpoints: weightedrouter2
132134
spec:
135+
terminationGracePeriodSeconds: 1
133136
containers:
134137
- name: test
135138
image: openshift/hello-openshift

0 commit comments

Comments
 (0)