Add doc values support for JSON fields. (#40069)

When `doc_values` are enabled, we now add two `SortedSetDocValuesFields` for each token: one containing the raw `value`, and another with `key\0value`. The root JSON field uses the standard `SortedSetDVOrdinalsIndexFieldData`. For keyed fields, this PR introduces a new type ` KeyedJsonIndexFieldData` that wraps the standard ordinals field data and filters out values that do not match the right prefix. This gives support for sorting on JSON fields, as well as simple keyword-style aggregations like `terms`.

One slightly tricky aspect is caching of these doc values. Given a keyed JSON field, we need to make sure we don't store values filtered on a certain prefix under the same cache key as ones filtered on a different prefix. However, we also want to load and cache global ordinals only once per keyed JSON field, as opposed to having a separate cache entry per prefix.

Author: jtibshirani

docs/reference/mapping/types/json.asciidoc

@@ -24,7 +24,7 @@ the following advantages:
 
 However, `json` fields present a trade-off in terms of search functionality.
 Only basic queries are allowed, with no support for numeric range queries or
-aggregations. Further information on the limitations can be found in the
+highlighting. Further information on the limitations can be found in the
 <<supported-operations, Supported operations>> section.
 
 NOTE: The `json` mapping type should **not** be used for indexing all JSON
@@ -107,9 +107,13 @@ Currently, `json` fields can be used with the following query types:
 
 When querying, it is not possible to refer to field keys using wildcards, as in
 `{ "term": {"labels.time*": 1541457010}}`. Note that all queries, including
-`range`, treat the values as string keywords.
+`range`, treat the values as string keywords. Highlighting is not supported on
+`json` fields.
 
-Aggregating, highlighting, or sorting on a `json` field is not supported.
+It is possible to sort on a `json` field, as well as perform simple
+keyword-style aggregations such as `terms`. As with queries, there is no
+special support for numerics -- all values in the JSON object are treated as
+keywords. When sorting, this implies that values are compared lexicographically.
 
 Finally, because of the way leaf values are stored in the index, the null
 character `\0` is not allowed to appear in the keys of the JSON object.
@@ -154,6 +158,18 @@ parameters are accepted:
     objects. If a JSON field exceeds this limit, then an error will be
     thrown. Defaults to `20`.
 
+<<doc-values,`doc_values`>>::
+
+    Should the field be stored on disk in a column-stride fashion, so that it
+    can later be used for sorting, aggregations, or scripting? Accepts `true`
+    (default) or `false`.
+
+<<eager-global-ordinals,`eager_global_ordinals`>>::
+
+    Should global ordinals be loaded eagerly on refresh? Accepts `true` or `false`
+    (default). Enabling this is a good idea on fields that are frequently used for
+    terms aggregations.
+
 <<ignore-above,`ignore_above`>>::
 
     Leaf values longer than this limit will not be indexed. By default, there

server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java

@@ -21,15 +21,19 @@
 
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.OrdinalMap;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -39,9 +43,21 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -93,7 +109,7 @@ private static class Defaults {
         static {
             FIELD_TYPE.setTokenized(false);
             FIELD_TYPE.setStored(false);
-            FIELD_TYPE.setHasDocValues(false);
+            FIELD_TYPE.setHasDocValues(true);
             FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
             FIELD_TYPE.setOmitNorms(true);
             FIELD_TYPE.freeze();
@@ -127,14 +143,6 @@ public Builder indexOptions(IndexOptions indexOptions) {
             return super.indexOptions(indexOptions);
         }
 
-        @Override
-        public Builder docValues(boolean docValues) {
-            if (docValues) {
-                throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields do not support doc values");
-            }
-            return super.docValues(docValues);
-        }
-
         public Builder depthLimit(int depthLimit) {
             if (depthLimit < 0) {
                 throw new IllegalArgumentException("[depth_limit] must be positive, got " + depthLimit);
@@ -143,6 +151,11 @@ public Builder depthLimit(int depthLimit) {
             return this;
         }
 
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            fieldType().setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
+        }
+
         public Builder ignoreAbove(int ignoreAbove) {
             if (ignoreAbove < 0) {
                 throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
@@ -166,11 +179,6 @@ public Builder copyTo(CopyTo copyTo) {
             throw new UnsupportedOperationException("[copy_to] is not supported for [" + CONTENT_TYPE + "] fields.");
         }
 
-        @Override
-        protected boolean defaultDocValues(Version indexCreated) {
-            return false;
-        }
-
         @Override
         public JsonFieldMapper build(BuilderContext context) {
             setupFieldType(context);
@@ -194,6 +202,9 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
                 if (propName.equals("depth_limit")) {
                     builder.depthLimit(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
+                } else if (propName.equals("eager_global_ordinals")) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
+                    iterator.remove();
                 } else if (propName.equals("ignore_above")) {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
@@ -221,7 +232,7 @@ public static final class KeyedJsonFieldType extends StringFieldType {
         private final String key;
         private boolean splitQueriesOnWhitespace;
 
-        KeyedJsonFieldType(String key) {
+        public KeyedJsonFieldType(String key) {
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
             setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
             this.key = key;
@@ -323,6 +334,7 @@ public Query wildcardQuery(String value,
                 CONTENT_TYPE + "] fields.");
         }
 
+        @Override
         public BytesRef indexedValueForSearch(Object value) {
             if (value == null) {
                 return null;
@@ -334,6 +346,108 @@ public BytesRef indexedValueForSearch(Object value) {
             String keyedValue = JsonFieldParser.createKeyedValue(key, stringValue);
             return new BytesRef(keyedValue);
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+            failIfNoDocValues();
+            return new KeyedJsonIndexFieldData.Builder(key);
+        }
+    }
+
+    /**
+     * A field data implementation that gives access to the values associated with
+     * a particular JSON key.
+     *
+     * This class wraps the field data that is built directly on the keyed JSON field, and
+     * filters out values whose prefix doesn't match the requested key. Loading and caching
+     * is fully delegated to the wrapped field data, so that different {@link KeyedJsonIndexFieldData}
+     * for the same JSON field share the same global ordinals.
+     */
+    public static class KeyedJsonIndexFieldData implements IndexOrdinalsFieldData {
+        private final String key;
+        private final IndexOrdinalsFieldData delegate;
+
+        private KeyedJsonIndexFieldData(String key, IndexOrdinalsFieldData delegate) {
+            this.delegate = delegate;
+            this.key = key;
+        }
+
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getFieldName() {
+            return delegate.getFieldName();
+        }
+
+        @Override
+        public SortField sortField(Object missingValue,
+                                   MultiValueMode sortMode,
+                                   XFieldComparatorSource.Nested nested,
+                                   boolean reverse) {
+            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+            return new SortField(getFieldName(), source, reverse);
+        }
+
+        @Override
+        public void clear() {
+            delegate.clear();
+        }
+
+        @Override
+        public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+            AtomicOrdinalsFieldData fieldData = delegate.load(context);
+            return new KeyedJsonAtomicFieldData(key, fieldData);
+        }
+
+        @Override
+        public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+            AtomicOrdinalsFieldData fieldData = delegate.loadDirect(context);
+            return new KeyedJsonAtomicFieldData(key, fieldData);
+        }
+
+        @Override
+        public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+            IndexOrdinalsFieldData fieldData = delegate.loadGlobal(indexReader);
+            return new KeyedJsonIndexFieldData(key, fieldData);
+        }
+
+        @Override
+        public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
+            IndexOrdinalsFieldData fieldData = delegate.localGlobalDirect(indexReader);
+            return new KeyedJsonIndexFieldData(key, fieldData);
+        }
+
+        @Override
+        public OrdinalMap getOrdinalMap() {
+            return delegate.getOrdinalMap();
+        }
+
+        @Override
+        public Index index() {
+            return delegate.index();
+        }
+
+        public static class Builder implements IndexFieldData.Builder {
+            private final String key;
+
+            Builder(String key) {
+                this.key = key;
+            }
+
+            @Override
+            public IndexFieldData<?> build(IndexSettings indexSettings,
+                                           MappedFieldType fieldType,
+                                           IndexFieldDataCache cache,
+                                           CircuitBreakerService breakerService,
+                                           MapperService mapperService) {
+                String fieldName = fieldType.name();
+                IndexOrdinalsFieldData delegate = new SortedSetDVOrdinalsIndexFieldData(indexSettings,
+                    cache, fieldName, breakerService, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+                return new KeyedJsonIndexFieldData(key, delegate);
+            }
+        }
     }
 
     /**
@@ -396,7 +510,11 @@ public Object valueForDisplay(Object value) {
 
         @Override
         public Query existsQuery(QueryShardContext context) {
-            return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
+            if (hasDocValues()) {
+                return new DocValuesFieldExistsQuery(name());
+            } else {
+                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
+            }
         }
 
         @Override
@@ -420,6 +538,12 @@ public Query wildcardQuery(String value,
             throw new UnsupportedOperationException("[wildcard] queries are not currently supported on [" +
                 CONTENT_TYPE + "] fields.");
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
     }
 
     private final JsonFieldParser fieldParser;
@@ -438,7 +562,7 @@ private JsonFieldMapper(String simpleName,
         this.depthLimit = depthLimit;
         this.ignoreAbove = ignoreAbove;
         this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(),
-            depthLimit, ignoreAbove, fieldType.nullValueAsString());
+            fieldType, depthLimit, ignoreAbove);
     }
 
     @Override
@@ -476,7 +600,9 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
             return;
         }
 
-        if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored()) {
+        if (fieldType.indexOptions() == IndexOptions.NONE
+                && !fieldType.hasDocValues()
+                && !fieldType.stored()) {
             context.parser().skipChildren();
             return;
         }
@@ -490,22 +616,22 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
             fields.add(new StoredField(fieldType.name(), storedValue));
         }
 
-        if (fieldType().indexOptions() != IndexOptions.NONE) {
-            XContentParser indexedFieldsParser = context.parser();
+        XContentParser indexedFieldsParser = context.parser();
 
-            // If store is enabled, we've already consumed the content to produce the stored field. Here we
-            // 'reset' the parser, so that we can traverse the content again.
-            if (storedValue != null) {
-                indexedFieldsParser = JsonXContent.jsonXContent.createParser(context.parser().getXContentRegistry(),
-                    context.parser().getDeprecationHandler(),
-                    storedValue.bytes);
-                indexedFieldsParser.nextToken();
-            }
-
-            fields.addAll(fieldParser.parse(indexedFieldsParser));
+        // If store is enabled, we've already consumed the content to produce the stored field. Here we
+        // 'reset' the parser, so that we can traverse the content again.
+        if (storedValue != null) {
+            indexedFieldsParser = JsonXContent.jsonXContent.createParser(context.parser().getXContentRegistry(),
+                context.parser().getDeprecationHandler(),
+                storedValue.bytes);
+            indexedFieldsParser.nextToken();
         }
 
-        createFieldNamesField(context, fields);
+        fields.addAll(fieldParser.parse(indexedFieldsParser));
+
+        if (!fieldType.hasDocValues()) {
+            createFieldNamesField(context, fields);
+        }
     }
 
     @Override