Limit result size (resulting_object_cnt_max)

Changes:
  - add two statistics to accessor_general:
    - returning_object_cnt
    - fetched_object_cnt
  - check `returning_object_cnt < returning_object_cnt_max` and
      `fetched_object_cnt < fetched_object_cnt_mx` after each tuple
      processed
  - add test for all accessors on new parameters

Closes #34

Author: Khatskevich

Makefile

@@ -4,7 +4,8 @@ default:
 .PHONY: lint
 lint:
 	luacheck graphql/*.lua test/local/*.lua test/testdata/*.lua \
-		test/common/lua/*.lua --no-redefined --no-unused-args
+		test/common/*test.lua test/common/lua/*.lua \
+		--no-redefined --no-unused-args
 
 .PHONY: test
 test: lint

graphql/accessor_general.lua

@@ -9,6 +9,8 @@ local avro_schema = require('avro_schema')
 local utils = require('graphql.utils')
 
 local accessor_general = {}
+local DEF_RESULTING_OBJECT_CNT_MAX = 10000
+local DEF_FETCHED_OBJECT_CNT_MAX = 10000
 
 --- Validate and compile set of avro schemas (with respect to service fields).
 --- @tparam table schemas map where keys are string names and values are
@@ -550,7 +552,8 @@ end
 ---
 --- * `count` (number),
 --- * `objs` (table, list of objects),
---- * `pivot_found` (boolean).
+--- * `pivot_found` (boolean),
+--- * `statistics` (table, per-query statistics).
 ---
 --- @tparam cdata tuple flatten representation of an object to process
 ---
@@ -562,7 +565,9 @@ end
 --- * `do_filter` (boolean, whether we need to filter out non-matching
 ---   objects),
 --- * `pivot_filter` (table, set of fields to match the objected pointed by
----   `offset` arqument of the GraphQL query)
+---   `offset` arqument of the GraphQL query),
+--- * `resulting_object_cnt_max` (number),
+--- * `fetched_object_cnt_max` (number).
 ---
 --- @return nil
 ---
@@ -580,6 +585,14 @@ local function process_tuple(state, tuple, opts)
     local filter = opts.filter
     local do_filter = opts.do_filter
     local pivot_filter = opts.pivot_filter
+    local qstats = state.statistics
+    local resulting_object_cnt_max = opts.resulting_object_cnt_max
+    local fetched_object_cnt_max = opts.fetched_object_cnt_max
+    qstats.fetched_object_cnt = qstats.fetched_object_cnt + 1
+    assert(qstats.fetched_object_cnt <= fetched_object_cnt_max,
+            ('fetched object count[%d] exceeds limit[%d] ' ..
+                    '(`fetched_object_cnt_max` in accessor)'):format(
+                    qstats.fetched_object_cnt, fetched_object_cnt_max))
 
     -- skip all items before pivot (the item pointed by offset)
     local obj = nil
@@ -604,10 +617,12 @@ local function process_tuple(state, tuple, opts)
     -- add the matching object, update count and check limit
     state.objs[#state.objs + 1] = obj
     state.count = state.count + 1
+    qstats.resulting_object_cnt = qstats.resulting_object_cnt + 1
+    assert(qstats.resulting_object_cnt <= resulting_object_cnt_max,
+            ('returning object count[%d] exceeds limit[%d] ' ..
+                    '(`resulting_object_cnt_max` in accessor)'):format(
+                    qstats.resulting_object_cnt, resulting_object_cnt_max))
     if limit ~= nil and state.count >= limit then
-        assert(limit == nil or state.count <= limit,
-            ('count[%d] exceeds limit[%s] (in for)'):format(
-            state.count, limit))
         return false
     end
     return true
@@ -631,8 +646,11 @@ end
 --- @tparam table args table of arguments passed within the query except ones
 --- that forms the `filter` parameter
 ---
+--- @tparam table extra table which contains extra information related to
+--- current select and the whole query
+---
 --- @treturn table list of matching objects
-local function select_internal(self, collection_name, from, filter, args)
+local function select_internal(self, collection_name, from, filter, args, extra)
     assert(type(self) == 'table',
         'self must be a table, got ' .. type(self))
     assert(type(collection_name) == 'string',
@@ -694,6 +712,7 @@ local function select_internal(self, collection_name, from, filter, args)
         count = 0,
         objs = {},
         pivot_found = false,
+        statistics = extra.qcontext.statistics
     }
 
     -- read only process_tuple options
@@ -703,6 +722,8 @@ local function select_internal(self, collection_name, from, filter, args)
         filter = filter,
         do_filter = not full_match,
         pivot_filter = nil, -- filled later if needed
+        resulting_object_cnt_max = self.settings.resulting_object_cnt_max,
+        fetched_object_cnt_max = self.settings.fetched_object_cnt_max
     }
 
     if index == nil then
@@ -791,7 +812,10 @@ end
 ---
 --- @tparam table opts `schemas`, `collections`, `service_fields` and `indexes`
 --- to give the data accessor all needed meta-information re data; the format is
---- shown below
+--- shown below; additional attributes `resulting_object_cnt_max` and
+--- `fetched_object_cnt_max` are optional positive numbers which help to control
+--- query behaviour in case it requires more resources than expected _(default
+--- value is 10,000)_
 ---
 --- @tparam table funcs set of functions (`is_collection_exists`, `get_index`,
 --- `get_primary_index`) allows this abstract data accessor behaves in the
@@ -837,6 +861,10 @@ function accessor_general.new(opts, funcs)
     local collections = opts.collections
     local service_fields = opts.service_fields
     local indexes = opts.indexes
+    local resulting_object_cnt_max = opts.resulting_object_cnt_max or
+                                     DEF_RESULTING_OBJECT_CNT_MAX
+    local fetched_object_cnt_max = opts.fetched_object_cnt_max or
+                                   DEF_FETCHED_OBJECT_CNT_MAX
 
     assert(type(schemas) == 'table',
         'schemas must be a table, got ' .. type(schemas))
@@ -846,6 +874,12 @@ function accessor_general.new(opts, funcs)
         'service_fields must be a table, got ' .. type(service_fields))
     assert(type(indexes) == 'table',
         'indexes must be a table, got ' .. type(indexes))
+    assert(type(resulting_object_cnt_max) == 'number' and
+                resulting_object_cnt_max > 0,
+        'resulting_object_cnt_max must be natural number')
+    assert(type(fetched_object_cnt_max) == 'number' and
+                fetched_object_cnt_max > 0,
+        'fetched_object_cnt_max must be natural number')
 
     local models = compile_schemas(schemas, service_fields)
     validate_collections(collections, schemas, indexes)
@@ -861,10 +895,14 @@ function accessor_general.new(opts, funcs)
         models = models,
         index_cache = index_cache,
         funcs = funcs,
+        settings = {
+            resulting_object_cnt_max = resulting_object_cnt_max,
+            fetched_object_cnt_max = fetched_object_cnt_max
+        }
     }, {
         __index = {
             select = function(self, parent, collection_name, from,
-                    filter, args)
+                    filter, args, extra)
                 assert(type(parent) == 'table',
                     'parent must be a table, got ' .. type(parent))
                 assert(from == nil or type(from) == 'table',
@@ -877,8 +915,13 @@ function accessor_general.new(opts, funcs)
                     'from must be nil or from.connection_name ' ..
                     'must be a string, got ' ..
                     type((from or {}).connection_name))
+                -- use `extra.qcontext` to store per-query variables
+                extra.qcontext.statistics = extra.qcontext.statistics or {
+                    resulting_object_cnt = 0,
+                    fetched_object_cnt = 0
+                }
                 return select_internal(self, collection_name, from, filter,
-                    args)
+                    args, extra)
             end,
             list_args = function(self, collection_name)
                 -- get name of field of primary key

graphql/tarantool_graphql.lua

@@ -286,7 +286,9 @@ gql_type = function(state, avro_schema, collection, collection_name)
                         destination_args_names = destination_args_names,
                         destination_args_values = destination_args_values,
                     }
-
+                    local extra = {
+                        qcontext = info.qcontext
+                    }
                     local object_args_instance = {} -- passed to 'filter'
                     local list_args_instance = {} -- passed to 'args'
                     for k, v in pairs(args_instance) do
@@ -302,7 +304,7 @@ gql_type = function(state, avro_schema, collection, collection_name)
                     end
                     local objs = accessor:select(parent,
                         c.destination_collection, from,
-                        object_args_instance, list_args_instance)
+                        object_args_instance, list_args_instance, extra)
                     assert(type(objs) == 'table',
                         'objs list received from an accessor ' ..
                         'must be a table, got ' .. type(objs))
@@ -397,8 +399,11 @@ local function parse_cfg(cfg)
                     end
                 end
                 local from = nil
+                local extra = {
+                    qcontext = info.qcontext
+                }
                 return accessor:select(rootValue, name, from,
-                    object_args_instance, list_args_instance)
+                    object_args_instance, list_args_instance, extra)
             end,
         }
     end
@@ -507,7 +512,7 @@ end
 ---     accessor = setmetatable({}, {
 ---         __index = {
 ---             select = function(self, parent, collection_name, from,
----                     object_args_instance, list_args_instance)
+---                     object_args_instance, list_args_instance, extra)
 ---                 -- from is nil for a top-level object, otherwise it is
 ---                 --
 ---                 -- {
@@ -517,6 +522,11 @@ end
 ---                 --     destination_args_values = <...>,
 ---                 -- }
 ---                 --
+---                 -- extra is a table which contains additional data for the
+---                 -- query; by now it consists of a single qcontext table,
+---                 -- which can be used by accessor to store any query-related
+---                 -- data
+---                 --
 ---                 return ...
 ---             end,
 ---             list_args = function(self, collection_name)

test/common/limit_result.test.lua

@@ -0,0 +1,86 @@
+#!/usr/bin/env tarantool
+local multirunner = require('multirunner')
+local data = require('test_data_user_order')
+local test_run = require('test_run').new()
+local tap = require('tap')
+local graphql = require('graphql')
+
+box.cfg({})
+local test = tap.test('result cnt')
+test:plan(6)
+
+-- require in-repo version of graphql/ sources despite current working directory
+local fio = require('fio')
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' .. package.path
+
+local function run(setup_name, shard)
+    print(setup_name)
+    local accessor_class
+    local virtbox
+    -- SHARD
+    if shard ~= nil then
+        accessor_class = graphql.accessor_shard
+        virtbox = shard
+    else
+        accessor_class = graphql.accessor_space
+        virtbox = box.space
+    end
+    local accessor = accessor_class.new({
+        schemas = data.meta.schemas,
+        collections = data.meta.collections,
+        service_fields = data.meta.service_fields,
+        indexes = data.meta.indexes,
+        resulting_object_cnt_max = 3,
+        fetched_object_cnt_max = 5
+    })
+
+    local gql_wrapper = graphql.new({
+        schemas = data.meta.schemas,
+        collections = data.meta.collections,
+        accessor = accessor,
+    })
+    data.fill_test_data(virtbox)
+    local query = [[
+        query object_result_max($user_id: Int, $description: String) {
+            user_collection(id: $user_id) {
+                id
+                last_name
+                first_name
+                order_connection(description: $description){
+                    id
+                    user_id
+                    description
+                }
+            }
+        }
+    ]]
+
+    local gql_query = gql_wrapper:compile(query)
+    local variables = {
+        user_id = 5,
+    }
+    local ok, result = pcall(gql_query.execute, gql_query, variables)
+    assert(ok == false, "this test should fail")
+    test:like(result,
+              'count%[4%] exceeds limit%[3%] %(`resulting_object_cnt_max`',
+              'resulting_object_cnt_max test')
+    variables = {
+        user_id = 5,
+        description = "no such description"
+    }
+    ok, result = pcall(gql_query.execute, gql_query, variables)
+    assert(ok == false, "this test should fail")
+    test:like(result,
+              'count%[6%] exceeds limit%[5%] %(`fetched_object_cnt_max`',
+              'resulting_object_cnt_max test')
+
+
+end
+
+multirunner.run(test_run,
+                data.init_spaces,
+                data.drop_spaces,
+                run)
+
+os.exit(test:check() == true and 0 or 1)