Support lookup by a partial set of index parts

* Fixes #30.
* Fixes #38.

Author: Totktonada

graphql/accessor_general.lua

@@ -121,14 +121,42 @@ local function filter_names_fingerprint(filter)
     return name_list_str
 end
 
+--- Get an index using parts tree built by @{build_index_parts_tree}.
+---
+--- @tparam table node root of the prefix tree for certain collection
+---
+--- @tparam table filter map of key-value to filter objects against
+---
+--- @treturn string `index_name` or `nil` is found index
+---
+--- @treturn number `len` is a number of index parts will be used at lookup
+local function get_best_matched_index(node, filter)
+    local index_name = (node.index_names or {})[1]
+    local max_len = 0
+
+    for k, v in pairs(filter) do
+        local successor_node = (node.successors or {})[k]
+        if successor_node ~= nil then
+            local new_filter = table.copy(filter)
+            new_filter[k] = nil
+            local branch_index_name, branch_len =
+                get_best_matched_index(successor_node, new_filter)
+            if branch_index_name ~= nil and branch_len > max_len then
+                index_name = branch_index_name
+                max_len = branch_len
+            end
+        end
+    end
+
+    return index_name, max_len + 1
+end
+
 -- XXX: raw idea: we can store field-to-field_no mapping when creating
 -- `lookup_index_name` to faster form the value_list
 
 --- Flatten filter values (transform to a list) against specific index to
 --- passing it to index:pairs().
 ---
---- Only full keys are supported for a compound index for now.
----
 --- @tparam table self the data accessor
 ---
 --- @tparam table filter filter for objects, its values will ordered to form
@@ -144,6 +172,9 @@ end
 --- passed index
 ---
 --- @treturn table `value_list` the value to pass to index:pairs()
+---
+--- @treturn table `new_filter` the `filter` value w/o values extracted to
+--- `value_list`
 local function flatten_filter(self, filter, collection_name, index_name)
     assert(type(self) == 'table',
         'self must be a table, got ' .. type(self))
@@ -153,6 +184,7 @@ local function flatten_filter(self, filter, collection_name, index_name)
         'index_name must be a string, got ' .. type(index_name))
 
     local value_list = {}
+    local new_filter = table.copy(filter)
 
     -- fill value_list
     local index_meta = self.indexes[collection_name][index_name]
@@ -163,6 +195,7 @@ local function flatten_filter(self, filter, collection_name, index_name)
         local value = filter[field_name]
         if value == nil then break end
         value_list[#value_list + 1] = value
+        new_filter[field_name] = nil
     end
 
     -- check for correctness: non-empty value_list
@@ -172,26 +205,11 @@ local function flatten_filter(self, filter, collection_name, index_name)
             json.encode(filter), index_name))
     end
 
-    -- check for correctness: all filter fields are used
-    local count = 0
-    for k, v in pairs(filter) do
-        count = count + 1
-    end
-    if count ~= #value_list then -- avoid extra json.encode()
-        assert(count ~= #value_list,
-            ('filter items count does not match index fields count: ' ..
-            'filter: %s, index_name: %s'):format(json.encode(filter),
-            index_name))
-    end
-
-    local full_match = #value_list == #index_meta.fields
-    return full_match, value_list
+    local full_match = #value_list == #index_meta.fields and
+        next(new_filter) == nil
+    return full_match, value_list, new_filter
 end
 
--- XXX: support partial match for primary/secondary indexes and support to skip
--- fields to get an index (full_match must be false in the case because
--- returned items will be additionally filtered after unflatten).
-
 --- Choose an index for lookup tuple(s) by a 'filter'. The filter holds fields
 --- values of object(s) we want to find. It uses prebuilt `lookup_index_name`
 --- table representing available indexes, which created by the
@@ -230,9 +248,12 @@ end
 ---
 --- @treturn string `index_name` is name of the found index or nil
 ---
---- @treturn table `value_list` is values list from the `filter` argument
---- ordered in the such way that can be passed to the found index (has some
---- meaning only when `index_name ~= nil`)
+--- @treturn table `new_filter` is the filter value w/o values extracted into
+--- `value_list`
+---
+--- @treturn table `value_list` (optional) is values list from the `filter`
+--- argument ordered in the such way that can be passed to the found index (has
+--- some meaning only when `index_name ~= nil`)
 ---
 --- @treturn table `pivot` (optional) an offset argument represented depending
 --- of a case: whether we'll lookup for the offset by an index; it is either
@@ -259,6 +280,10 @@ local get_index_name = function(self, collection_name, from, filter, args)
     assert(type(lookup_index_name) == 'table',
         'lookup_index_name must be a table, got ' .. type(lookup_index_name))
 
+    local parts_tree = index_cache.parts_tree
+    assert(type(parts_tree) == 'table',
+        'parts_tree must be a table, got ' .. type(parts_tree))
+
     local connection_indexes = index_cache.connection_indexes
     assert(type(connection_indexes) == 'table',
         'connection_indexes must be a table, got ' .. type(connection_indexes))
@@ -276,6 +301,7 @@ local get_index_name = function(self, collection_name, from, filter, args)
         assert(connection_type ~= nil, 'connection_type must not be nil')
         local full_match = connection_type == '1:1' and next(filter) == nil
         local value_list = from.destination_args_values
+        local new_filter = filter
 
         local pivot
         if args.offset ~= nil then
@@ -296,21 +322,22 @@ local get_index_name = function(self, collection_name, from, filter, args)
             pivot = {filter = pivot_filter}
         end
 
-        return full_match, index_name, value_list, pivot
+        return full_match, index_name, new_filter, value_list, pivot
     end
 
     -- The 'fast offset' case. Here we fetch top-level objects starting from
     -- passed offset. Select will be performed by the primary index and
     -- corresponding offset in `pivot.value_list`, then the result will be
-    -- postprocessed using `filter`, if necessary.
+    -- postprocessed using `new_filter`, if necessary.
     if args.offset ~= nil then
         local index_name, index_meta = get_primary_index_meta(self,
             collection_name)
         local full_match
         local pivot_value_list
+        local new_filter = filter
         if type(args.offset) == 'table' then
-            full_match, pivot_value_list = flatten_filter(self, args.offset,
-                collection_name, index_name)
+            full_match, pivot_value_list, new_filter = flatten_filter(self,
+                args.offset, collection_name, index_name)
             assert(full_match == true, 'offset by a partial key is forbidden')
         else
             assert(#index_meta.fields == 1,
@@ -320,22 +347,34 @@ local get_index_name = function(self, collection_name, from, filter, args)
         end
         local pivot = {value_list = pivot_value_list}
         full_match = full_match and next(filter) == nil
-        return full_match, index_name, filter, pivot
+        return full_match, index_name, new_filter, nil, pivot
     end
 
     -- The 'no offset' case. Here we fetch top-level object either by found
     -- index or using full scan (if the index was not found).
+
+    -- try to find full index
     local name_list_str = filter_names_fingerprint(filter)
     assert(lookup_index_name[collection_name] ~= nil,
         ('cannot find any index for collection "%s"'):format(collection_name))
     local index_name = lookup_index_name[collection_name][name_list_str]
     local full_match = false
     local value_list = nil
+    local new_filter = filter
+
+    -- try to find partial index
+    if index_name == nil then
+        local root = parts_tree[collection_name]
+        index_name = get_best_matched_index(root, filter)
+    end
+
+    -- fill full_match and value_list appropriatelly
     if index_name ~= nil then
-        full_match, value_list = flatten_filter(self, filter, collection_name,
-            index_name)
+        full_match, value_list, new_filter = flatten_filter(self, filter,
+            collection_name, index_name)
     end
-    return full_match, index_name, value_list
+
+    return full_match, index_name, new_filter, value_list
 end
 
 --- Build `lookup_index_name` table (part of `index_cache`) to use in the
@@ -402,6 +441,34 @@ local function build_lookup_index_name(indexes)
     return lookup_index_name
 end
 
+--- Build `parts_tree` to use in @{get_index_name} for lookup best matching
+--- index.
+---
+--- @tparam table indexes indexes metainformation as defined in the @{new}
+--- function
+---
+--- @treturn table `roots` resulting parts prefix tree
+local function build_index_parts_tree(indexes)
+    local roots = {}
+
+    for collection_name, indexes_meta in pairs(indexes) do
+        local root = {}
+        roots[collection_name] = root
+        for index_name, index_meta in pairs(indexes_meta) do
+            local cur = root
+            for _, field in ipairs(index_meta.fields) do
+                cur.successors = cur.successors or {}
+                cur.successors[field] = cur.successors[field] or {}
+                cur = cur.successors[field]
+                cur.index_names = cur.index_names or {}
+                cur.index_names[#cur.index_names + 1] = index_name
+            end
+        end
+    end
+
+    return roots
+end
+
 --- Build `connection_indexes` table (part of `index_cache`) to use in the
 --- @{get_index_name} function.
 ---
@@ -491,6 +558,7 @@ end
 local function build_index_cache(indexes, collections)
     return {
         lookup_index_name = build_lookup_index_name(indexes),
+        parts_tree = build_index_parts_tree(indexes),
         connection_indexes = build_connection_indexes(indexes, collections),
     }
 end
@@ -682,8 +750,8 @@ local function select_internal(self, collection_name, from, filter, args, extra)
         ('cannot find collection "%s"'):format(collection_name))
 
     -- search for suitable index
-    local full_match, index_name, index_value, pivot = get_index_name(
-        self, collection_name, from, filter, args)
+    local full_match, index_name, filter, index_value, pivot = get_index_name(
+        self, collection_name, from, filter, args) -- we redefine filter here
     local index = index_name ~= nil and
         self.funcs.get_index(collection_name, index_name) or nil
     if from ~= nil then

test/local/space_compound_index.result

@@ -7,6 +7,11 @@ user_collection:
   user_num: 12
 ...
 
+RESULT
+---
+user_collection: []
+...
+
 RESULT
 ---
 user_collection:
@@ -117,6 +122,16 @@ user_collection:
   user_num: 20
 ...
 
+RESULT
+---
+user_collection: []
+...
+
+RESULT
+---
+user_collection: []
+...
+
 RESULT
 ---
 user_collection:
@@ -157,6 +172,16 @@ user_collection:
   last_name: last name b
 ...
 
+RESULT
+---
+user_collection:
+- order_connection: []
+  user_str: user_str_b
+  first_name: first name b
+  user_num: 12
+  last_name: last name b
+...
+
 RESULT
 ---
 user_collection:

test/shard_no_redundancy/shard_compound_index.result

@@ -123,6 +123,11 @@ testdata.run_queries(gql_wrapper)
     user_num: 12
   ...
 
+  RESULT
+  ---
+  user_collection: []
+  ...
+
   RESULT
   ---
   user_collection:
@@ -233,6 +238,16 @@ testdata.run_queries(gql_wrapper)
     user_num: 20
   ...
 
+  RESULT
+  ---
+  user_collection: []
+  ...
+
+  RESULT
+  ---
+  user_collection: []
+  ...
+
   RESULT
   ---
   user_collection:
@@ -273,6 +288,16 @@ testdata.run_queries(gql_wrapper)
     last_name: last name b
   ...
 
+  RESULT
+  ---
+  user_collection:
+  - order_connection: []
+    user_str: user_str_b
+    first_name: first name b
+    user_num: 12
+    last_name: last name b
+  ...
+
   RESULT
   ---
   user_collection:

test/shard_redundancy/shard_compound_index.result

@@ -141,6 +141,11 @@ testdata.run_queries(gql_wrapper)
     user_num: 12
   ...
 
+  RESULT
+  ---
+  user_collection: []
+  ...
+
   RESULT
   ---
   user_collection:
@@ -251,6 +256,16 @@ testdata.run_queries(gql_wrapper)
     user_num: 20
   ...
 
+  RESULT
+  ---
+  user_collection: []
+  ...
+
+  RESULT
+  ---
+  user_collection: []
+  ...
+
   RESULT
   ---
   user_collection:
@@ -291,6 +306,16 @@ testdata.run_queries(gql_wrapper)
     last_name: last name b
   ...
 
+  RESULT
+  ---
+  user_collection:
+  - order_connection: []
+    user_str: user_str_b
+    first_name: first name b
+    user_num: 12
+    last_name: last name b
+  ...
+
   RESULT
   ---
   user_collection: