Skip to content

Commit 2d5e8e5

Browse files
rcrichtonrcrichton
authored
Merge pull request #1200 from jembi/prometheus-exporter
Prometheus exporter for OpenHIM
2 parents 63850b1 + 67ff4c0 commit 2d5e8e5

File tree

9 files changed

+4165
-2749
lines changed

9 files changed

+4165
-2749
lines changed

‎grafana-dashboards/openhim_nodejs_dashboard.json

+978
Large diffs are not rendered by default.

‎grafana-dashboards/openhim_transactions_dashboard.json

+715
Large diffs are not rendered by default.

‎package-lock.json

+2,211-2,747
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
"passport-local": "^1.0.0",
7979
"passport-openidconnect": "^0.1.1",
8080
"pem": "^1.14.4",
81+
"prom-client": "^14.2.0",
8182
"raw-body": "^2.4.1",
8283
"semver": "^7.3.2",
8384
"ssl-root-cas": "1.3.1",

‎src/api/metrics.js

+5
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import logger from 'winston'
44
import moment from 'moment'
55
import mongoose from 'mongoose'
6+
import {register} from 'prom-client'
67

78
import * as authorisation from './authorisation'
89
import * as metrics from '../metrics'
@@ -79,3 +80,7 @@ function calculateAverage(total, count) {
7980
}
8081
return total / count
8182
}
83+
84+
export async function getPrometheusMetrics(ctx) {
85+
ctx.body = await register.metrics()
86+
}

‎src/koaApi.js

+16-2
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ export function setupApp(done) {
4040

4141
// Configure Sessions Middleware
4242
app.keys = [config.api.sessionKey]
43-
43+
4444
if (config.api.trustProxy) {
4545
app.proxy = true
4646
}
47-
47+
4848
app.use(
4949
session(
5050
{
@@ -107,6 +107,20 @@ export function setupApp(done) {
107107
// @deprecated: Token authentication
108108
app.use(route.get('/authenticate/:username', users.authenticateToken))
109109

110+
// Publicly exposed Prometheus metrics endpoint
111+
// only invoked if the accept header is correct else passes to the next middleware
112+
app.use(async (ctx, next) => {
113+
if (
114+
ctx.request.headers['accept']?.includes('application/openmetrics-text') &&
115+
ctx.request.path === '/metrics' &&
116+
ctx.request.method === 'GET'
117+
) {
118+
metrics.getPrometheusMetrics(ctx)
119+
} else {
120+
await next()
121+
}
122+
})
123+
110124
// Authenticate the API request
111125
app.use(authentication.authenticate)
112126

‎src/metrics.js

+72
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,21 @@
11
'use strict'
22

33
import moment from 'moment'
4+
import {collectDefaultMetrics, Counter, Histogram} from 'prom-client'
5+
import {ChannelModelAPI} from './model/channels'
6+
import {ClientModelAPI} from './model/clients'
7+
8+
collectDefaultMetrics({prefix: 'openhim_'})
9+
const txCounter = new Counter({
10+
name: 'openhim_transactions_total',
11+
help: 'Total transactions processed',
12+
labelNames: ['method', 'status', 'client', 'channel', 'code']
13+
})
14+
const respTimeHistogram = new Histogram({
15+
name: 'openhim_request_duration',
16+
help: 'Request response time in seconds',
17+
labelNames: ['method', 'status', 'client', 'channel', 'code']
18+
})
419

520
import {
621
METRIC_TYPE_DAY,
@@ -19,6 +34,14 @@ const TRANSACTION_STATUS_KEYS = {
1934

2035
const METRIC_UPDATE_OPTIONS = {upsert: true, setDefaultsOnInsert: true}
2136

37+
const cache = {
38+
clientMap: {},
39+
clientsLastFetch: moment(0),
40+
channelMap: {},
41+
channelsLastFetch: moment(0),
42+
refreshMins: 1
43+
}
44+
2245
async function recordTransactionMetric(fields, update) {
2346
return MetricModel.updateOne(
2447
fields,
@@ -27,6 +50,42 @@ async function recordTransactionMetric(fields, update) {
2750
)
2851
}
2952

53+
async function getClientNameFromCache(clientID) {
54+
if (
55+
cache.clientsLastFetch.isBefore(
56+
moment().subtract(cache.refreshMins, 'minute')
57+
) ||
58+
cache.clientMap[clientID] === undefined
59+
) {
60+
const clients = await ClientModelAPI.find({}, {name: 1})
61+
cache.clientMap = clients.reduce((clientMap, client) => {
62+
clientMap[client._id.toString()] = client.name
63+
return clientMap
64+
}, {})
65+
cache.clientsLastFetch = moment()
66+
}
67+
68+
return cache.clientMap[clientID]
69+
}
70+
71+
async function getChannelNameFromCache(channelID) {
72+
if (
73+
cache.channelsLastFetch.isBefore(
74+
moment().subtract(cache.refreshMins, 'minute')
75+
) ||
76+
cache.channelMap[channelID] === undefined
77+
) {
78+
const channels = await ChannelModelAPI.find({}, {name: 1})
79+
cache.channelMap = channels.reduce((channelMap, channel) => {
80+
channelMap[channel._id.toString()] = channel.name
81+
return channelMap
82+
}, {})
83+
cache.channelsLastFetch = moment()
84+
}
85+
86+
return cache.channelMap[channelID]
87+
}
88+
3089
export async function recordTransactionMetrics(transaction) {
3190
if (
3291
!transaction.response ||
@@ -43,6 +102,19 @@ export async function recordTransactionMetrics(transaction) {
43102
transaction.response.timestamp.getTime() -
44103
transaction.request.timestamp.getTime()
45104
const statusKey = TRANSACTION_STATUS_KEYS[transaction.status]
105+
106+
// collect metric for Prometheus
107+
const labels = {
108+
status: transaction.status,
109+
method: transaction.request?.method,
110+
client: await getClientNameFromCache(transaction.clientID),
111+
channel: await getChannelNameFromCache(transaction.channelID),
112+
code: transaction.response?.status
113+
}
114+
txCounter.inc(labels)
115+
respTimeHistogram.observe(labels, responseTime)
116+
117+
// collect metrics for internal metric API
46118
const update = {
47119
$inc: {
48120
requests: 1,

‎test/integration/metricsAPITests.js

+17
Original file line numberDiff line numberDiff line change
@@ -195,4 +195,21 @@ describe('API Metrics Tests', () =>
195195
.expect(400)
196196
})
197197
})
198+
199+
describe('*getPrometheusMetrics()', () => {
200+
it('should PUBLICLY fetch prometheus metrics and return custom metric types defined', async () => {
201+
const res = await request(BASE_URL)
202+
.get('/metrics')
203+
.set('Accept', 'application/openmetrics-text')
204+
.expect(200)
205+
206+
should.exist(res.text)
207+
res.text
208+
.includes('# TYPE openhim_transactions_total counter')
209+
.should.be.true('should contain openhim_transactions_total metric')
210+
res.text
211+
.includes('# TYPE openhim_request_duration histogram')
212+
.should.be.true('should contain openhim_request_duration metric')
213+
})
214+
})
198215
}))

‎test/unit/metricsTest.js

+150
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,24 @@
55

66
import should from 'should'
77
import {ObjectId} from 'mongodb'
8+
import {register} from 'prom-client'
89

910
import * as metrics from '../../src/metrics'
1011
import {MetricModel} from '../../src/model'
12+
import {ChannelModelAPI} from '../../src/model/channels'
13+
import {ClientModelAPI} from '../../src/model/clients'
1114

1215
describe('recordTransactionMetrics', () => {
1316
beforeEach(async () => {
1417
await MetricModel.deleteMany()
1518
})
1619

20+
after(async () => {
21+
await MetricModel.deleteMany()
22+
await ChannelModelAPI.deleteMany()
23+
await ClientModelAPI.deleteMany()
24+
})
25+
1726
it('should record the correct metrics for a transaction', async () => {
1827
const channelID = new ObjectId()
1928
const transaction = {
@@ -234,6 +243,147 @@ describe('recordTransactionMetrics', () => {
234243
const count = await MetricModel.countDocuments()
235244
should.equal(count, 0)
236245
})
246+
247+
it('should capture prometheus metrics to the default registry (undefined channel and client case)', async () => {
248+
const channelID = new ObjectId()
249+
const clientID = new ObjectId()
250+
const transaction = {
251+
status: 'Successful',
252+
channelID,
253+
clientID,
254+
request: {
255+
method: 'GET',
256+
timestamp: new Date('2017-12-07T09:17:58.333Z')
257+
},
258+
response: {
259+
timestamp: new Date('2017-12-07T09:18:01.500Z'),
260+
status: '200'
261+
}
262+
}
263+
264+
register.resetMetrics()
265+
await metrics.recordTransactionMetrics(transaction)
266+
267+
const txString = await register.getSingleMetricAsString(
268+
'openhim_transactions_total'
269+
)
270+
should.equal(
271+
txString,
272+
`# HELP openhim_transactions_total Total transactions processed
273+
# TYPE openhim_transactions_total counter
274+
openhim_transactions_total{status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 1`
275+
)
276+
277+
const reqString = await register.getSingleMetricAsString(
278+
'openhim_request_duration'
279+
)
280+
should.equal(
281+
reqString,
282+
`# HELP openhim_request_duration Request response time in seconds
283+
# TYPE openhim_request_duration histogram
284+
openhim_request_duration_bucket{le="0.005",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
285+
openhim_request_duration_bucket{le="0.01",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
286+
openhim_request_duration_bucket{le="0.025",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
287+
openhim_request_duration_bucket{le="0.05",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
288+
openhim_request_duration_bucket{le="0.1",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
289+
openhim_request_duration_bucket{le="0.25",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
290+
openhim_request_duration_bucket{le="0.5",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
291+
openhim_request_duration_bucket{le="1",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
292+
openhim_request_duration_bucket{le="2.5",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
293+
openhim_request_duration_bucket{le="5",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
294+
openhim_request_duration_bucket{le="10",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 0
295+
openhim_request_duration_bucket{le="+Inf",status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 1
296+
openhim_request_duration_sum{status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 3167
297+
openhim_request_duration_count{status="Successful",method="GET",client="undefined",channel="undefined",code="200"} 1`
298+
)
299+
})
300+
301+
it('should capture prometheus metrics to the default registry (existing channel and client case)', async () => {
302+
const channel1 = {
303+
name: 'TestChannel1',
304+
urlPattern: 'test/sample',
305+
allow: ['PoC', 'Test1', 'Test2'],
306+
routes: [
307+
{
308+
name: 'test route',
309+
host: 'localhost',
310+
port: 9876,
311+
primary: true
312+
}
313+
],
314+
txViewAcl: 'aGroup',
315+
updatedBy: {
316+
id: new ObjectId(),
317+
name: 'Test'
318+
}
319+
}
320+
const channel = await new ChannelModelAPI(channel1).save()
321+
322+
const testAppDoc = {
323+
clientID: 'testApp',
324+
clientDomain: 'test-client.jembi.org',
325+
name: 'TEST Client',
326+
roles: ['OpenMRS_PoC', 'PoC'],
327+
passwordAlgorithm: 'sha512',
328+
passwordHash:
329+
'28dce3506eca8bb3d9d5a9390135236e8746f15ca2d8c86b8d8e653da954e9e3632bf9d85484ee6e9b28a3ada30eec89add42012b185bd9a4a36a07ce08ce2ea',
330+
passwordSalt: '1234567890',
331+
cert: ''
332+
}
333+
const client = await new ClientModelAPI(testAppDoc).save()
334+
335+
const transaction = {
336+
status: 'Successful',
337+
channelID: channel._id,
338+
clientID: client._id,
339+
request: {
340+
method: 'GET',
341+
timestamp: new Date('2017-12-07T09:17:58.333Z')
342+
},
343+
response: {
344+
timestamp: new Date('2017-12-07T09:18:01.500Z'),
345+
status: '200'
346+
}
347+
}
348+
349+
register.resetMetrics()
350+
await metrics.recordTransactionMetrics(transaction)
351+
// record second transaction to cover cache retrieval
352+
await metrics.recordTransactionMetrics(transaction)
353+
354+
const txString = await register.getSingleMetricAsString(
355+
'openhim_transactions_total'
356+
)
357+
should.equal(
358+
txString,
359+
`# HELP openhim_transactions_total Total transactions processed
360+
# TYPE openhim_transactions_total counter
361+
openhim_transactions_total{status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 2`
362+
)
363+
364+
const reqString = await register.getSingleMetricAsString(
365+
'openhim_request_duration'
366+
)
367+
should.equal(
368+
reqString,
369+
`# HELP openhim_request_duration Request response time in seconds
370+
# TYPE openhim_request_duration histogram
371+
openhim_request_duration_bucket{le="0.005",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
372+
openhim_request_duration_bucket{le="0.01",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
373+
openhim_request_duration_bucket{le="0.025",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
374+
openhim_request_duration_bucket{le="0.05",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
375+
openhim_request_duration_bucket{le="0.1",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
376+
openhim_request_duration_bucket{le="0.25",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
377+
openhim_request_duration_bucket{le="0.5",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
378+
openhim_request_duration_bucket{le="1",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
379+
openhim_request_duration_bucket{le="2.5",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
380+
openhim_request_duration_bucket{le="5",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
381+
openhim_request_duration_bucket{le="10",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 0
382+
openhim_request_duration_bucket{le="+Inf",status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 2
383+
openhim_request_duration_sum{status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 6334
384+
openhim_request_duration_count{status="Successful",method="GET",client="${client.name}",channel="${channel.name}",code="200"} 2`
385+
)
386+
})
237387
})
238388

239389
describe('calculateMetrics', () => {

0 commit comments

Comments
 (0)