Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit b57efb6

Browse files
TotktonadaTotktonada
committed
Rework statistics, count fetches and cache hits
We count a request across a cluster as one fetch and don't count requests to individual storages.
1 parent ea84bf6 commit b57efb6

File tree

5 files changed

+111
-60
lines changed

5 files changed

+111
-60
lines changed

‎graphql/accessor_general.lua

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -854,9 +854,19 @@ local function process_tuple(self, state, tuple, opts)
854854
local pivot_filter = opts.pivot_filter
855855
local qcontext = state.qcontext
856856

857+
assert(opts.fetches_cnt ~= nil)
858+
local full_scan_cnt = opts.is_full_scan and opts.fetches_cnt or 0
859+
local index_lookup_cnt = opts.is_full_scan and 0 or opts.fetches_cnt
857860
qcontext.statistics:objects_fetched({
858-
is_full_scan = opts.is_full_scan,
859-
objects_count = opts.fetched_tuples_cnt,
861+
fetches_cnt = opts.fetches_cnt,
862+
fetched_objects_cnt = opts.fetched_tuples_cnt,
863+
full_scan_cnt = full_scan_cnt,
864+
index_lookup_cnt = index_lookup_cnt,
865+
})
866+
867+
qcontext.statistics:cache_lookup({
868+
cache_hits_cnt = opts.cache_hits_cnt,
869+
cache_hit_objects_cnt = opts.cache_hit_tuples_cnt,
860870
})
861871

862872
if clock.monotonic64() > qcontext.deadline_clock then
@@ -914,7 +924,7 @@ local function process_tuple(self, state, tuple, opts)
914924
state.count = state.count + 1
915925

916926
qcontext.statistics:objects_retired({
917-
objects_count = 1,
927+
retired_objects_cnt = 1,
918928
})
919929

920930
if limit ~= nil and state.count >= limit then
@@ -1159,9 +1169,18 @@ local function invoke_select_internal(self, prepared_select)
11591169
end
11601170

11611171
for _, tuple in iterable:pairs(index_value, iterator_opts, out) do
1162-
local fetched_tuples_cnt = out.fetched_tuples_cnt
1163-
check(out.fetched_tuples_cnt, 'out.fetched_tuples_cnt', 'number')
1172+
local fetches_cnt = out.fetches_cnt or 0
1173+
local fetched_tuples_cnt = out.fetched_tuples_cnt or 0
1174+
local cache_hits_cnt = out.cache_hits_cnt or 0
1175+
local cache_hit_tuples_cnt = out.cache_hit_tuples_cnt or 0
1176+
check(fetches_cnt, 'fetches_cnt', 'number')
1177+
check(fetched_tuples_cnt, 'fetched_tuples_cnt', 'number')
1178+
check(cache_hits_cnt, 'cache_hits_cnt', 'number')
1179+
check(cache_hit_tuples_cnt, 'cache_hit_tuples_cnt', 'number')
1180+
out.fetches_cnt = 0
11641181
out.fetched_tuples_cnt = 0
1182+
out.cache_hits_cnt = 0
1183+
out.cache_hit_tuples_cnt = 0
11651184

11661185
tuple_count = tuple_count + 1
11671186

@@ -1172,8 +1191,11 @@ local function invoke_select_internal(self, prepared_select)
11721191
'%d tuples'):format(extra.exp_tuple_count))
11731192
end
11741193

1175-
select_opts.fetched_tuples_cnt = fetched_tuples_cnt
11761194
select_opts.is_full_scan = is_full_scan
1195+
select_opts.fetches_cnt = fetches_cnt
1196+
select_opts.fetched_tuples_cnt = fetched_tuples_cnt
1197+
select_opts.cache_hits_cnt = cache_hits_cnt
1198+
select_opts.cache_hit_tuples_cnt = cache_hit_tuples_cnt
11771199
local continue = process_tuple(self, select_state, tuple,
11781200
select_opts)
11791201
if not continue then break end
@@ -1664,18 +1686,23 @@ function accessor_general.new(opts, funcs)
16641686
end
16651687

16661688
local fetch_id = res.fetch_id
1667-
local stats = res.stats
1689+
local stat = res.stat
16681690
check(fetch_id, 'fetch_id', 'number')
1669-
check(stats, 'stats', 'table')
1691+
check(stat, 'stat', 'table')
1692+
check(stat.fetches_cnt, 'stat.fetches_cnt', 'number')
1693+
check(stat.fetched_tuples_cnt, 'stat.fetched_tuples_cnt',
1694+
'number')
1695+
check(stat.full_scan_cnt, 'stat.full_scan_cnt', 'number')
1696+
check(stat.index_lookup_cnt, 'stat.index_lookup_cnt', 'number')
16701697

16711698
-- update statistics
16721699
init_qcontext(self, qcontext)
1673-
for _, stat in ipairs(stats) do
1674-
qcontext.statistics:objects_fetched({
1675-
is_full_scan = stat.is_full_scan,
1676-
objects_count = stat.fetched_tuples_cnt,
1677-
})
1678-
end
1700+
qcontext.statistics:objects_fetched({
1701+
fetches_cnt = stat.fetches_cnt,
1702+
fetched_objects_cnt = stat.fetched_tuples_cnt,
1703+
full_scan_cnt = stat.full_scan_cnt,
1704+
index_lookup_cnt = stat.index_lookup_cnt
1705+
})
16791706

16801707
-- XXX: check timeout
16811708

‎graphql/accessor_shard.lua

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,6 @@ local function get_index(self, collection_name, index_name)
184184
local index = setmetatable({}, {
185185
__index = {
186186
pairs = function(_, value, opts, out)
187-
out.fetched_tuples_cnt = 0
188187
local func_name = 'accessor_shard.get_index.<index>.pairs'
189188

190189
-- perform select
@@ -194,7 +193,8 @@ local function get_index(self, collection_name, index_name)
194193
index_name, opts, value, 0)
195194
accessor_shard_helpers.shard_check_error(func_name,
196195
tuples, err)
197-
out.fetched_tuples_cnt = out.fetched_tuples_cnt + #tuples
196+
out.fetches_cnt = 1
197+
out.fetched_tuples_cnt = #tuples
198198

199199
-- create iterator
200200
local cur = 1
@@ -487,12 +487,11 @@ end
487487
---
488488
--- {
489489
--- fetch_id = <number>, -- identifier of the fetched data
490-
--- stats = { -- data to update statistics
491-
--- {
492-
--- is_full_scan = <boolean>,
493-
--- fetched_tuples_cnt = <number>,
494-
--- },
495-
--- ...
490+
--- stat = { -- data to update statistics
491+
--- fetches_cnt = <number>,
492+
--- fetched_tuples_cnt = <number>,
493+
--- full_scan_cnt = <number>,
494+
--- index_lookup_cnt = <number>,
496495
--- }
497496
--- }
498497
local function cache_fetch(self, batches)

‎graphql/accessor_shard_cache.lua

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,12 @@ local function cache_fetch(self, batches)
5050
local fetch_id = self.fetch_id_next
5151
self.fetch_id_next = self.fetch_id_next + 1
5252

53-
local stats = {}
53+
local stat = {
54+
fetches_cnt = 0,
55+
fetched_tuples_cnt = 0,
56+
full_scan_cnt = 0,
57+
index_lookup_cnt = 0,
58+
}
5459

5560
for _, batch in pairs(batches) do
5661
local ids = get_select_ids(self, batch, {skip_cached = true})
@@ -82,10 +87,19 @@ local function cache_fetch(self, batches)
8287
for _, tuple in ipairs(node_result) do
8388
table.insert(results[j], tuple)
8489
end
85-
stats[j] = stats[j] or {
86-
is_full_scan = batch.keys[j] == nil,
87-
fetched_tuples_cnt = 0,
88-
}
90+
end
91+
end
92+
93+
-- count fetches: assume a fetch as one request across cluster (don't
94+
-- count individulal requests to storages)
95+
stat.fetches_cnt = stat.fetches_cnt + 1
96+
-- count full scan and index lookup counts
97+
for i, key in ipairs(batch.keys) do
98+
if key == nil or (type(key) == 'table' and
99+
next(key) == nil) then
100+
stat.full_scan_cnt = stat.full_scan_cnt + 1
101+
else
102+
stat.index_lookup_cnt = stat.index_lookup_cnt + 1
89103
end
90104
end
91105

@@ -94,8 +108,9 @@ local function cache_fetch(self, batches)
94108
-- sort by a primary key
95109
local primary_index_info = accessor_shard_index_info.get_index_info(
96110
batch.collection_name, 0)
97-
for j, result in ipairs(results) do
98-
stats[j].fetched_tuples_cnt = stats[j].fetched_tuples_cnt + #result
111+
for _, result in ipairs(results) do
112+
-- count fetched tuples
113+
stat.fetched_tuples_cnt = stat.fetched_tuples_cnt + #result
99114
table.sort(result, function(a, b)
100115
for i, part in pairs(primary_index_info.parts) do
101116
if a[part.fieldno] ~= b[part.fieldno] then
@@ -129,7 +144,7 @@ local function cache_fetch(self, batches)
129144

130145
return {
131146
fetch_id = fetch_id,
132-
stats = stats,
147+
stat = stat,
133148
}
134149
end
135150

@@ -193,8 +208,8 @@ local function cache_lookup(self, collection_name, index_name, key,
193208
'iterator_opts in cache_lookup call and :pairs() on ' ..
194209
'returned iterable')
195210

196-
-- we actually do not fetch any tuples here
197-
out.fetched_tuples_cnt = 0
211+
out.cache_hits_cnt = 1
212+
out.cache_hit_tuples_cnt = #self.tuples
198213

199214
-- create iterator
200215
local cur = 1

‎graphql/accessor_space.lua

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ local function get_index(self, collection_name, index_name)
3939
return setmetatable({}, {
4040
__index = {
4141
pairs = function(_, value, opts, out)
42+
out.fetches_cnt = 1
4243
out.fetched_tuples_cnt = 0
4344
local gen, param, state = index:pairs(value, opts)
4445
local function new_gen(param, state)

‎graphql/statistics.lua

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -10,28 +10,22 @@ local statistics = {}
1010
local function objects_fetched(self, info)
1111
check(self, 'self', 'table')
1212
check(info, 'info', 'table')
13-
local is_full_scan = info.is_full_scan
14-
local objects_count = info.objects_count
15-
check(is_full_scan, 'is_full_scan', 'boolean')
16-
check(objects_count, 'objects_count', 'number')
13+
local fetches_cnt = info.fetches_cnt
14+
local fetched_objects_cnt = info.fetched_objects_cnt
15+
local full_scan_cnt = info.full_scan_cnt
16+
local index_lookup_cnt = info.index_lookup_cnt
17+
check(fetches_cnt, 'fetches_cnt', 'number')
18+
check(fetched_objects_cnt, 'fetched_objects_cnt', 'number')
19+
check(full_scan_cnt, 'full_scan_cnt', 'number')
20+
check(index_lookup_cnt, 'index_lookup_cnt', 'number')
1721

18-
if objects_count == 0 then return end
22+
-- count fetches and fetched objects
23+
self.fetches_cnt = self.fetches_cnt + fetches_cnt
24+
self.fetched_object_cnt = self.fetched_object_cnt + fetched_objects_cnt
1925

20-
-- count fetched objects
21-
self.fetched_object_cnt = self.fetched_object_cnt + objects_count
22-
23-
-- count all select request
24-
self.select_requests_cnt = self.select_requests_cnt + 1
25-
26-
if is_full_scan then
27-
-- count full scan select request
28-
self.full_scan_select_requests_cnt =
29-
self.full_scan_select_requests_cnt + 1
30-
else
31-
-- count index select request
32-
self.index_select_requests_cnt =
33-
self.index_select_requests_cnt + 1
34-
end
26+
-- count full scan and index select request
27+
self.full_scan_cnt = self.full_scan_cnt + full_scan_cnt
28+
self.index_lookup_cnt = self.index_lookup_cnt + index_lookup_cnt
3529

3630
if self.limits.fetched_object_cnt_max == nil then
3731
return
@@ -48,12 +42,10 @@ end
4842
local function objects_retired(self, info)
4943
check(self, 'self', 'table')
5044
check(info, 'info', 'table')
51-
local objects_count = info.objects_count
52-
check(objects_count, 'objects_count', 'number')
45+
local retired_objects_cnt = info.retired_objects_cnt
46+
check(retired_objects_cnt, 'retired_objects_cnt', 'number')
5347

54-
if objects_count == 0 then return end
55-
56-
self.resulting_object_cnt = self.resulting_object_cnt + objects_count
48+
self.resulting_object_cnt = self.resulting_object_cnt + retired_objects_cnt
5749

5850
if self.limits.resulting_object_cnt_max == nil then
5951
return
@@ -66,17 +58,33 @@ local function objects_retired(self, info)
6658
end
6759
end
6860

61+
--- Count cache hit / miss event.
62+
local function cache_lookup(self, info)
63+
check(self, 'self', 'table')
64+
check(info, 'info', 'table')
65+
local cache_hits_cnt = info.cache_hits_cnt
66+
local cache_hit_objects_cnt = info.cache_hit_objects_cnt
67+
check(cache_hits_cnt, 'cache_hits_cnt', 'number')
68+
check(cache_hit_objects_cnt, 'cache_hit_objects_cnt', 'number')
69+
70+
self.cache_hits_cnt = self.cache_hits_cnt + cache_hits_cnt
71+
self.cache_hit_objects_cnt = self.cache_hit_objects_cnt +
72+
cache_hit_objects_cnt
73+
end
74+
6975
function statistics.new(opts)
7076
local opts = opts or {}
7177
local resulting_object_cnt_max = opts.resulting_object_cnt_max
7278
local fetched_object_cnt_max = opts.fetched_object_cnt_max
7379

7480
return setmetatable({
7581
resulting_object_cnt = 0, -- retire
82+
fetches_cnt = 0, -- fetch
7683
fetched_object_cnt = 0, -- fetch
77-
select_requests_cnt = 0, -- fetch
78-
full_scan_select_requests_cnt = 0, -- fetch
79-
index_select_requests_cnt = 0, -- fetch
84+
full_scan_cnt = 0, -- fetch
85+
index_lookup_cnt = 0, -- fetch
86+
cache_hits_cnt = 0, -- cache lookup
87+
cache_hit_objects_cnt = 0, -- cache lookup
8088
limits = {
8189
resulting_object_cnt_max = resulting_object_cnt_max, -- retire limit
8290
fetched_object_cnt_max = fetched_object_cnt_max, -- fetch limit
@@ -85,6 +93,7 @@ function statistics.new(opts)
8593
__index = {
8694
objects_fetched = objects_fetched,
8795
objects_retired = objects_retired,
96+
cache_lookup = cache_lookup,
8897
}
8998
})
9099
end

0 commit comments

Comments
 (0)