From d1490e184a22dd5c781192a75b99533a96f0deff Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 13 Dec 2016 19:33:32 +0100 Subject: [PATCH 1/4] Format doc values fields. Currently `docvalues_fields` return the values of the fields as they are stored in doc values. I don't like that it exposes implementation details, but there are also user-facing issues like the fact it cannot work with binary fields. This change will also make it easier for users to reindex if they do not store the source, since `docvalues_fields` will return data is such a format that it can be put in an indexing request with the same mappings. The hard part of the change is backward compatibility, since it is breaking. The approach taken here is that 5.x will keep exposing the internal representation, with a special format name called `use_field_format` which will format the field depending on how it is mapped. This will become the default in 6.0, and this hardcoded format name will be removed in 7.0 to ease the transition from 5.x to 6.x. --- .../action/search/ExpandSearchPhase.java | 2 +- .../action/search/SearchRequestBuilder.java | 14 +- .../index/mapper/BinaryFieldMapper.java | 7 + .../index/query/InnerHitBuilder.java | 65 +++++-- .../elasticsearch/search/DocValueFormat.java | 166 ++++++++++++------ .../elasticsearch/search/SearchModule.java | 1 + .../histogram/InternalDateHistogram.java | 4 +- .../bucket/histogram/InternalHistogram.java | 4 +- .../bucket/range/InternalBinaryRange.java | 8 +- .../bucket/range/InternalRange.java | 8 +- .../significant/SignificantLongTerms.java | 4 +- .../significant/SignificantStringTerms.java | 2 +- .../SignificantTermsAggregatorFactory.java | 4 +- .../bucket/terms/DoubleTerms.java | 4 +- .../aggregations/bucket/terms/LongTerms.java | 4 +- .../bucket/terms/StringTerms.java | 2 +- .../InternalNumericMetricsAggregation.java | 4 +- .../aggregations/metrics/avg/InternalAvg.java | 2 +- .../aggregations/metrics/max/InternalMax.java | 2 +- .../aggregations/metrics/min/InternalMin.java | 2 +- .../hdr/AbstractInternalHDRPercentiles.java | 4 +- .../AbstractInternalTDigestPercentiles.java | 4 +- .../metrics/stats/InternalStats.java | 8 +- .../stats/extended/InternalExtendedStats.java | 4 +- .../aggregations/metrics/sum/InternalSum.java | 2 +- .../tophits/TopHitsAggregationBuilder.java | 59 ++++--- .../tophits/TopHitsAggregatorFactory.java | 7 +- .../pipeline/InternalSimpleValue.java | 2 +- .../InternalBucketMetricValue.java | 2 +- .../percentile/InternalPercentilesBucket.java | 2 +- .../derivative/InternalDerivative.java | 2 +- .../search/builder/SearchSourceBuilder.java | 64 +++++-- .../fetch/subphase/DocValueFieldsContext.java | 120 ++++++++++++- .../subphase/DocValueFieldsFetchSubPhase.java | 80 +++++++-- .../index/mapper/BinaryFieldTypeTests.java | 9 + .../index/mapper/BooleanFieldTypeTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 5 +- .../search/DocValueFormatTests.java | 52 +++--- .../search/fields/SearchFieldsIT.java | 80 ++++++++- .../migration/migrate_6_0/search.asciidoc | 31 ++++ .../search/request/docvalue-fields.asciidoc | 64 +++++++ .../test/search/10_source_filtering.yaml | 12 +- 42 files changed, 719 insertions(+), 207 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 1ec21ab424901..ba39f8db43b40 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -129,7 +129,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio } } if (options.getDocValueFields() != null) { - options.getDocValueFields().forEach(groupSource::docValueField); + options.getDocValueFields().forEach(field -> groupSource.docValueField(field.getName(), field.getFormat())); } if (options.getStoredFieldsContext() != null && options.getStoredFieldsContext().fieldNames() != null) { options.getStoredFieldsContext().fieldNames().forEach(groupSource::storedField); diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index ffe2c1b20c516..9cb9a339f7157 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -284,7 +284,7 @@ public SearchRequestBuilder setFetchSource(@Nullable String[] includes, @Nullabl /** * Adds a docvalue based field to load and return. The field does not have to be stored, - * but its recommended to use non analyzed or numeric fields. + * but its recommended to use non analyzed fields. * * @param name The field to get from the docvalue */ @@ -293,6 +293,18 @@ public SearchRequestBuilder addDocValueField(String name) { return this; } + /** + * Adds a docvalue based field to load and return. The field does not have to be stored, + * but its recommended to use non analyzed fields. + * + * @param name The field to get from the docvalue + * @param format How to format the field, {@code null} to use defaults. + */ + public SearchRequestBuilder addDocValueField(String name, String format) { + sourceBuilder().docValueField(name, format); + return this; + } + /** * Adds a stored field to load and return (note, it must be stored) as part of the search request. */ diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index aa60bfcd9a043..20f8c3b4b1be0 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -36,6 +36,8 @@ import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.DocValueFormat; +import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.Base64; @@ -120,6 +122,11 @@ public BytesReference valueForDisplay(Object value) { return bytes; } + @Override + public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) { + return DocValueFormat.BINARY; + } + @Override public IndexFieldData.Builder fielddataBuilder() { failIfNoDocValues(); diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index c1b240066abe2..a28ca5e3f3974 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -54,6 +55,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.XContentParser.Token.END_OBJECT; @@ -81,7 +83,9 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " + "if the field is not stored"); }, SearchSourceBuilder.FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY); - PARSER.declareStringArray(InnerHitBuilder::setDocValueFields, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD); + PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields, + DocValueFieldsContext.Field::fromXContent, + DocValueFieldsContext.DOCVALUE_FIELDS_FIELD); PARSER.declareField((p, i, c) -> { try { Set scriptFields = new HashSet<>(); @@ -144,7 +148,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl private StoredFieldsContext storedFieldsContext; private QueryBuilder query = DEFAULT_INNER_HIT_QUERY; private List> sorts; - private List docValueFields; + private List docValueFields; private Set scriptFields; private HighlightBuilder highlightBuilder; private FetchSourceContext fetchSourceContext; @@ -221,7 +225,21 @@ public InnerHitBuilder(StreamInput in) throws IOException { version = in.readBoolean(); trackScores = in.readBoolean(); storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new); - docValueFields = (List) in.readGenericValue(); + if (in.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + List fieldNameList = (List) in.readGenericValue(); + if (fieldNameList == null) { + docValueFields = null; + } else { + docValueFields = new ArrayList<>(); + for (String name : fieldNameList) { + docValueFields.add(new DocValueFieldsContext.Field(name, null)); + } + } + } else if (in.readBoolean()) { + docValueFields = in.readList(DocValueFieldsContext.Field::new); + } else { + docValueFields = null; + } if (in.readBoolean()) { int size = in.readVInt(); scriptFields = new HashSet<>(size); @@ -262,7 +280,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(version); out.writeBoolean(trackScores); out.writeOptionalWriteable(storedFieldsContext); - out.writeGenericValue(docValueFields); + if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + out.writeBoolean(docValueFields != null); + if (docValueFields != null) { + out.writeList(docValueFields); + } + } else { + out.writeGenericValue(docValueFields == null ? null : docValueFields.stream().map( + DocValueFieldsContext.Field::getName).collect(Collectors.toList())); + } boolean hasScriptFields = scriptFields != null; out.writeBoolean(hasScriptFields); if (hasScriptFields) { @@ -413,7 +439,7 @@ public InnerHitBuilder setStoredFieldNames(List fieldNames) { * @deprecated Use {@link InnerHitBuilder#getDocValueFields()} instead. */ @Deprecated - public List getFieldDataFields() { + public List getFieldDataFields() { return docValueFields; } @@ -423,7 +449,7 @@ public List getFieldDataFields() { * @deprecated Use {@link InnerHitBuilder#setDocValueFields(List)} instead. */ @Deprecated - public InnerHitBuilder setFieldDataFields(List fieldDataFields) { + public InnerHitBuilder setFieldDataFields(List fieldDataFields) { this.docValueFields = fieldDataFields; return this; } @@ -435,24 +461,20 @@ public InnerHitBuilder setFieldDataFields(List fieldDataFields) { */ @Deprecated public InnerHitBuilder addFieldDataField(String field) { - if (docValueFields == null) { - docValueFields = new ArrayList<>(); - } - docValueFields.add(field); - return this; + return addDocValueField(field); } /** * Gets the docvalue fields. */ - public List getDocValueFields() { + public List getDocValueFields() { return docValueFields; } /** * Sets the stored fields to load from the docvalue and return. */ - public InnerHitBuilder setDocValueFields(List docValueFields) { + public InnerHitBuilder setDocValueFields(List docValueFields) { this.docValueFields = docValueFields; return this; } @@ -461,10 +483,19 @@ public InnerHitBuilder setDocValueFields(List docValueFields) { * Adds a field to load from the docvalue and return. */ public InnerHitBuilder addDocValueField(String field) { + return addDocValueField(field, null); + } + + /** + * Adds a field to load from the docvalue and return. + * @param name name of the field + * @param format how to format the field, or {@code null} to use the defaults + */ + public InnerHitBuilder addDocValueField(String name, @Nullable String format) { if (docValueFields == null) { docValueFields = new ArrayList<>(); } - docValueFields.add(field); + docValueFields.add(new DocValueFieldsContext.Field(name, format)); return this; } @@ -673,9 +704,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws storedFieldsContext.toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder); } if (docValueFields != null) { - builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName()); - for (String fieldDataField : docValueFields) { - builder.value(fieldDataField); + builder.startArray(DocValueFieldsContext.DOCVALUE_FIELDS_FIELD.getPreferredName()); + for (DocValueFieldsContext.Field fieldDataField : docValueFields) { + fieldDataField.toXContent(builder); } builder.endArray(); } diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java index eb76db3be687b..7dbbc3fc252e1 100644 --- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -30,6 +30,8 @@ import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.joda.time.DateTimeZone; import java.io.IOException; @@ -39,6 +41,7 @@ import java.text.NumberFormat; import java.text.ParseException; import java.util.Arrays; +import java.util.Base64; import java.util.Locale; import java.util.Objects; import java.util.function.LongSupplier; @@ -46,32 +49,32 @@ /** A formatter for values as returned by the fielddata/doc-values APIs. */ public interface DocValueFormat extends NamedWriteable { - /** Format a long value. This is used by terms and histogram aggregations - * to format keys for fields that use longs as a doc value representation - * such as the {@code long} and {@code date} fields. */ - String format(long value); + /** Format a long value. This is used to convert the internal representation + * that is used by doc values back to the original type of the object. + * The return value must be supported by {@link XContentBuilder#value(Object)}. */ + Object format(long value); - /** Format a double value. This is used by terms and stats aggregations - * to format keys for fields that use numbers as a doc value representation - * such as the {@code long}, {@code double} or {@code date} fields. */ - String format(double value); + /** Format a double value. This is used to convert the internal representation + * that is used by doc values back to the original type of the object. + * The return value must be supported by {@link XContentBuilder#value(Object)}. */ + Object format(double value); - /** Format a double value. This is used by terms aggregations to format - * keys for fields that use binary doc value representations such as the - * {@code keyword} and {@code ip} fields. */ - String format(BytesRef value); + /** Format a binary value. This is used to convert the internal representation + * that is used by doc values back to the original type of the object. + * The return value must be supported by {@link XContentBuilder#value(Object)}. */ + Object format(BytesRef value); - /** Parse a value that was formatted with {@link #format(long)} back to the - * original long value. */ - long parseLong(String value, boolean roundUp, LongSupplier now); + /** Parse a value that comes from {@link XContentParser#objectBytes()} to the + * internal representation that is used by doc values. */ + long parseLong(Object value, boolean roundUp, LongSupplier now); - /** Parse a value that was formatted with {@link #format(double)} back to - * the original double value. */ - double parseDouble(String value, boolean roundUp, LongSupplier now); + /** Parse a value that comes from {@link XContentParser#objectBytes()} to the + * internal representation that is used by doc values. */ + double parseDouble(Object value, boolean roundUp, LongSupplier now); - /** Parse a value that was formatted with {@link #format(BytesRef)} back - * to the original BytesRef. */ - BytesRef parseBytesRef(String value); + /** Parse a value that comes from {@link XContentParser#objectBytes()} to the + * internal representation that is used by doc values. */ + BytesRef parseBytesRef(Object value); DocValueFormat RAW = new DocValueFormat() { @@ -85,13 +88,13 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public String format(long value) { - return Long.toString(value); + public Long format(long value) { + return value; } @Override - public String format(double value) { - return Double.toString(value); + public Double format(double value) { + return value; } @Override @@ -100,8 +103,8 @@ public String format(BytesRef value) { } @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { - double d = Double.parseDouble(value); + public long parseLong(Object value, boolean roundUp, LongSupplier now) { + double d = parseDouble(value, roundUp, now); if (roundUp) { d = Math.ceil(d); } else { @@ -111,13 +114,17 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) { } @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { - return Double.parseDouble(value); + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } else { + return Double.parseDouble(value.toString()); + } } @Override - public BytesRef parseBytesRef(String value) { - return new BytesRef(value); + public BytesRef parseBytesRef(Object value) { + return new BytesRef(value.toString()); } }; @@ -166,17 +173,17 @@ public String format(BytesRef value) { } @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { - return parser.parse(value, now, roundUp, timeZone); + public long parseLong(Object value, boolean roundUp, LongSupplier now) { + return parser.parse(value.toString(), now, roundUp, timeZone); } @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { return parseLong(value, roundUp, now); } @Override - public BytesRef parseBytesRef(String value) { + public BytesRef parseBytesRef(Object value) { throw new UnsupportedOperationException(); } } @@ -193,12 +200,12 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public String format(long value) { + public Object format(long value) { return GeoHashUtils.stringEncode(value); } @Override - public String format(double value) { + public Object format(double value) { return format((long) value); } @@ -208,17 +215,17 @@ public String format(BytesRef value) { } @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { + public long parseLong(Object value, boolean roundUp, LongSupplier now) { throw new UnsupportedOperationException(); } @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { throw new UnsupportedOperationException(); } @Override - public BytesRef parseBytesRef(String value) { + public BytesRef parseBytesRef(Object value) { throw new UnsupportedOperationException(); } }; @@ -235,13 +242,13 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public String format(long value) { - return java.lang.Boolean.valueOf(value != 0).toString(); + public Boolean format(long value) { + return java.lang.Boolean.valueOf(value != 0); } @Override - public String format(double value) { - return java.lang.Boolean.valueOf(value != 0).toString(); + public Boolean format(double value) { + return java.lang.Boolean.valueOf(value != 0); } @Override @@ -250,8 +257,8 @@ public String format(BytesRef value) { } @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { - switch (value) { + public long parseLong(Object value, boolean roundUp, LongSupplier now) { + switch (value.toString()) { case "false": return 0; case "true": @@ -261,12 +268,12 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) { } @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { return parseLong(value, roundUp, now); } @Override - public BytesRef parseBytesRef(String value) { + public BytesRef parseBytesRef(Object value) { throw new UnsupportedOperationException(); } }; @@ -296,22 +303,24 @@ public String format(double value) { public String format(BytesRef value) { byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); InetAddress inet = InetAddressPoint.decode(bytes); + // We do not return the inet address directly since XContentBuilder does not know + // how to deal with it return NetworkAddress.format(inet); } @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { + public long parseLong(Object value, boolean roundUp, LongSupplier now) { throw new UnsupportedOperationException(); } @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { throw new UnsupportedOperationException(); } @Override - public BytesRef parseBytesRef(String value) { - return new BytesRef(InetAddressPoint.encode(InetAddresses.forString(value))); + public BytesRef parseBytesRef(Object value) { + return new BytesRef(InetAddressPoint.encode(InetAddresses.forString(value.toString()))); } }; @@ -358,10 +367,10 @@ public String format(BytesRef value) { } @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { + public long parseLong(Object value, boolean roundUp, LongSupplier now) { Number n; try { - n = format.parse(value); + n = format.parse(value.toString()); } catch (ParseException e) { throw new RuntimeException(e); } @@ -379,10 +388,10 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) { } @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { Number n; try { - n = format.parse(value); + n = format.parse(value.toString()); } catch (ParseException e) { throw new RuntimeException(e); } @@ -390,7 +399,7 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) { } @Override - public BytesRef parseBytesRef(String value) { + public BytesRef parseBytesRef(Object value) { throw new UnsupportedOperationException(); } @@ -411,4 +420,47 @@ public int hashCode() { return Objects.hash(pattern); } } + + DocValueFormat BINARY = new DocValueFormat() { + + @Override + public String getWriteableName() { + return "binary"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + } + + @Override + public Long format(long value) { + throw new UnsupportedOperationException(); + } + + @Override + public Double format(double value) { + throw new UnsupportedOperationException(); + } + + @Override + public String format(BytesRef value) { + return Base64.getEncoder().encodeToString( + Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length)); + } + + @Override + public long parseLong(Object value, boolean roundUp, LongSupplier now) { + throw new UnsupportedOperationException(); + } + + @Override + public double parseDouble(Object value, boolean roundUp, LongSupplier now) { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef parseBytesRef(Object value) { + return new BytesRef(Base64.getDecoder().decode(value.toString())); + } + }; } diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index 8707d851d3a1c..9a0e7c4dba593 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -626,6 +626,7 @@ private void registerValueFormats() { registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH); registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP); registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW); + registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY); } /** diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java index dc05ab51e8ad7..4e9794bec901e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java @@ -104,7 +104,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String getKeyAsString() { - return format.format(key); + return format.format(key).toString(); } @Override @@ -135,7 +135,7 @@ Bucket reduce(List buckets, ReduceContext context) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - String keyAsString = format.format(key); + String keyAsString = format.format(key).toString(); if (keyed) { builder.startObject(keyAsString); } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java index c6d2aa9eeb959..d82c4e13d8c99 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java @@ -100,7 +100,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String getKeyAsString() { - return format.format(key); + return format.format(key).toString(); } @Override @@ -131,7 +131,7 @@ Bucket reduce(List buckets, ReduceContext context) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - String keyAsString = format.format(key); + String keyAsString = format.format(key).toString(); if (keyed) { builder.startObject(keyAsString); } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java index 640b1cfb467f4..2c717735547ba 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java @@ -124,9 +124,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (keyed) { if (key == null) { StringBuilder keyBuilder = new StringBuilder(); - keyBuilder.append(from == null ? "*" : format.format(from)); + keyBuilder.append(from == null ? "*" : format.format(from).toString()); keyBuilder.append("-"); - keyBuilder.append(to == null ? "*" : format.format(to)); + keyBuilder.append(to == null ? "*" : format.format(to).toString()); key = keyBuilder.toString(); } builder.startObject(key); @@ -155,7 +155,7 @@ public Object getFrom() { @Override public String getFromAsString() { - return from == null ? null : format.format(from); + return from == null ? null : format.format(from).toString(); } @Override @@ -165,7 +165,7 @@ public Object getTo() { @Override public String getToAsString() { - return to == null ? null : format.format(to); + return to == null ? null : format.format(to).toString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java index 3d0cf1b5a8c7d..bbdbde9ae5f74 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java @@ -97,7 +97,7 @@ public String getFromAsString() { if (Double.isInfinite(from)) { return null; } else { - return format.format(from); + return format.format(from).toString(); } } @@ -106,7 +106,7 @@ public String getToAsString() { if (Double.isInfinite(to)) { return null; } else { - return format.format(to); + return format.format(to).toString(); } } @@ -146,13 +146,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (!Double.isInfinite(from)) { builder.field(CommonFields.FROM.getPreferredName(), from); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.FROM_AS_STRING.getPreferredName(), format.format(from)); + builder.field(CommonFields.FROM_AS_STRING.getPreferredName(), format.format(from).toString()); } } if (!Double.isInfinite(to)) { builder.field(CommonFields.TO.getPreferredName(), to); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.TO_AS_STRING.getPreferredName(), format.format(to)); + builder.field(CommonFields.TO_AS_STRING.getPreferredName(), format.format(to).toString()); } } builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java index 6a714a6b035d6..404b27c7beaff 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java @@ -83,7 +83,7 @@ int compareTerm(SignificantTerms.Bucket other) { @Override public String getKeyAsString() { - return format.format(term); + return format.format(term).toString(); } @Override @@ -101,7 +101,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(CommonFields.KEY.getPreferredName(), term); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term)); + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString()); } builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount()); builder.field("score", score); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java index d62213bc47aca..d2cb46dc682c4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java @@ -88,7 +88,7 @@ int compareTerm(SignificantTerms.Bucket other) { @Override public String getKeyAsString() { - return format.format(termBytes); + return format.format(termBytes).toString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java index c27cabf78b2aa..8fb2e807a131a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java @@ -143,12 +143,12 @@ private long getBackgroundFrequency(String value) throws IOException { } public long getBackgroundFrequency(BytesRef termBytes) throws IOException { - String value = config.format().format(termBytes); + String value = config.format().format(termBytes).toString(); return getBackgroundFrequency(value); } public long getBackgroundFrequency(long termNum) throws IOException { - String value = config.format().format(termNum); + String value = config.format().format(termNum).toString(); return getBackgroundFrequency(value); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java index ae18cb59d9523..099f328ad12ae 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java @@ -62,7 +62,7 @@ protected void writeTermTo(StreamOutput out) throws IOException { @Override public String getKeyAsString() { - return format.format(term); + return format.format(term).toString(); } @Override @@ -89,7 +89,7 @@ Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError) { protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { builder.field(CommonFields.KEY.getPreferredName(), term); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term)); + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java index 98aa4825ee7a3..18a601332a202 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java @@ -62,7 +62,7 @@ protected void writeTermTo(StreamOutput out) throws IOException { @Override public String getKeyAsString() { - return format.format(term); + return format.format(term).toString(); } @Override @@ -89,7 +89,7 @@ Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError) { protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { builder.field(CommonFields.KEY.getPreferredName(), term); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term)); + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java index b48c443fac93a..0aa3a1e841c4f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java @@ -71,7 +71,7 @@ public Number getKeyAsNumber() { @Override public String getKeyAsString() { - return format.format(termBytes); + return format.format(termBytes).toString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java index dba16397fc050..b3439671580e7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java @@ -48,7 +48,7 @@ protected SingleValue(StreamInput in) throws IOException { @Override public String getValueAsString() { - return format.format(value()); + return format.format(value()).toString(); } @Override @@ -79,7 +79,7 @@ protected MultiValue(StreamInput in) throws IOException { public abstract double value(String name); public String valueAsString(String name) { - return format.format(value(name)); + return format.format(value(name)).toString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java index d6fe2e6a93e45..075ee586daeb8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java @@ -98,7 +98,7 @@ public InternalAvg doReduce(List aggregations, ReduceContex public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.field(CommonFields.VALUE.getPreferredName(), count != 0 ? getValue() : null); if (count != 0 && format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(getValue())); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(getValue()).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java index 112d379362700..449351b88b169 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java @@ -85,7 +85,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th boolean hasValue = !Double.isInfinite(max); builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? max : null); if (hasValue && format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(max)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(max).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java index dcf180dde89a4..886642c222baf 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java @@ -85,7 +85,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th boolean hasValue = !Double.isInfinite(min); builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? min : null); if (hasValue && format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(min)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(min).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java index 29b8c38062cdd..9c2b24fa7dc1e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java @@ -121,7 +121,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th double value = value(keys[i]); builder.field(key, value); if (format != DocValueFormat.RAW) { - builder.field(key + "_as_string", format.format(value)); + builder.field(key + "_as_string", format.format(value).toString()); } } builder.endObject(); @@ -133,7 +133,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.field(CommonFields.KEY.getPreferredName(), keys[i]); builder.field(CommonFields.VALUE.getPreferredName(), value); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); } builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java index 687b5533ecb31..e209114e1367a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java @@ -104,7 +104,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th double value = value(keys[i]); builder.field(key, value); if (format != DocValueFormat.RAW) { - builder.field(key + "_as_string", format.format(value)); + builder.field(key + "_as_string", format.format(value).toString()); } } builder.endObject(); @@ -116,7 +116,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.field(CommonFields.KEY.getPreferredName(), keys[i]); builder.field(CommonFields.VALUE.getPreferredName(), value); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); } builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java index 08c9292d54e62..a447574166f76 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java @@ -187,10 +187,10 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.field(Fields.AVG, count != 0 ? getAvg() : null); builder.field(Fields.SUM, count != 0 ? sum : null); if (count != 0 && format != DocValueFormat.RAW) { - builder.field(Fields.MIN_AS_STRING, format.format(min)); - builder.field(Fields.MAX_AS_STRING, format.format(max)); - builder.field(Fields.AVG_AS_STRING, format.format(getAvg())); - builder.field(Fields.SUM_AS_STRING, format.format(sum)); + builder.field(Fields.MIN_AS_STRING, format.format(min).toString()); + builder.field(Fields.MAX_AS_STRING, format.format(max).toString()); + builder.field(Fields.AVG_AS_STRING, format.format(getAvg()).toString()); + builder.field(Fields.SUM_AS_STRING, format.format(sum).toString()); } otherStatsToXCotent(builder, params); return builder; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java index 370399bfbb8db..f1fbe33788499 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java @@ -179,8 +179,8 @@ protected XContentBuilder otherStatsToXCotent(XContentBuilder builder, Params pa .endObject(); if (count != 0 && format != DocValueFormat.RAW) { - builder.field(Fields.SUM_OF_SQRS_AS_STRING, format.format(sumOfSqrs)); - builder.field(Fields.VARIANCE_AS_STRING, format.format(getVariance())); + builder.field(Fields.SUM_OF_SQRS_AS_STRING, format.format(sumOfSqrs).toString()); + builder.field(Fields.VARIANCE_AS_STRING, format.format(getVariance()).toString()); builder.field(Fields.STD_DEVIATION_AS_STRING, getStdDeviationAsString()); builder.startObject(Fields.STD_DEVIATION_BOUNDS_AS_STRING) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java index 576f1ea52cf6b..8fc968c3fd8cd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java @@ -84,7 +84,7 @@ public InternalSum doReduce(List aggregations, ReduceContex public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.field(CommonFields.VALUE.getPreferredName(), sum); if (format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(sum)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(sum).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java index 68a16b4d3c94c..9e2a605597a7d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.tophits; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -37,6 +38,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; @@ -67,7 +69,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder> sorts = null; private HighlightBuilder highlightBuilder; private StoredFieldsContext storedFieldsContext; - private List fieldDataFields; + private List fieldDataFields; private Set scriptFields; private FetchSourceContext fetchSourceContext; @@ -82,12 +84,16 @@ public TopHitsAggregationBuilder(StreamInput in) throws IOException { super(in); explain = in.readBoolean(); fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new); - if (in.readBoolean()) { - int size = in.readVInt(); - fieldDataFields = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - fieldDataFields.add(in.readString()); + if (in.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + if (in.readBoolean()) { + int size = in.readVInt(); + fieldDataFields = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + fieldDataFields.add(new DocValueFieldsContext.Field(in.readString(), null)); + } } + } else if (in.readBoolean()) { + fieldDataFields = in.readList(DocValueFieldsContext.Field::new); } storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new); from = in.readVInt(); @@ -118,9 +124,13 @@ protected void doWriteTo(StreamOutput out) throws IOException { boolean hasFieldDataFields = fieldDataFields != null; out.writeBoolean(hasFieldDataFields); if (hasFieldDataFields) { - out.writeVInt(fieldDataFields.size()); - for (String fieldName : fieldDataFields) { - out.writeString(fieldName); + if (out.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + out.writeVInt(fieldDataFields.size()); + for (DocValueFieldsContext.Field field : fieldDataFields) { + out.writeString(field.getName()); + } + } else { + out.writeList(fieldDataFields); } } out.writeOptionalWriteable(storedFieldsContext); @@ -380,13 +390,21 @@ public StoredFieldsContext storedFields() { * the search request. */ public TopHitsAggregationBuilder fieldDataField(String fieldDataField) { + return fieldDataField(fieldDataField, null); + } + + /** + * Adds a field to load from the field data cache and return as part of + * the search request. + */ + public TopHitsAggregationBuilder fieldDataField(String fieldDataField, String format) { if (fieldDataField == null) { throw new IllegalArgumentException("[fieldDataField] must not be null: [" + name + "]"); } if (fieldDataFields == null) { fieldDataFields = new ArrayList<>(); } - fieldDataFields.add(fieldDataField); + fieldDataFields.add(new DocValueFieldsContext.Field(fieldDataField, format)); return this; } @@ -394,7 +412,7 @@ public TopHitsAggregationBuilder fieldDataField(String fieldDataField) { * Adds fields to load from the field data cache and return as part of * the search request. */ - public TopHitsAggregationBuilder fieldDataFields(List fieldDataFields) { + public TopHitsAggregationBuilder fieldDataFields(List fieldDataFields) { if (fieldDataFields == null) { throw new IllegalArgumentException("[fieldDataFields] must not be null: [" + name + "]"); } @@ -408,7 +426,7 @@ public TopHitsAggregationBuilder fieldDataFields(List fieldDataFields) { /** * Gets the field-data fields. */ - public List fieldDataFields() { + public List fieldDataFields() { return fieldDataFields; } @@ -564,9 +582,9 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param storedFieldsContext.toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder); } if (fieldDataFields != null) { - builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName()); - for (String fieldDataField : fieldDataFields) { - builder.value(fieldDataField); + builder.startArray(DocValueFieldsContext.DOCVALUE_FIELDS_FIELD.getPreferredName()); + for (DocValueFieldsContext.Field fieldDataField : fieldDataFields) { + fieldDataField.toXContent(builder); } builder.endArray(); } @@ -682,15 +700,10 @@ public static TopHitsAggregationBuilder parse(String aggregationName, QueryParse if (SearchSourceBuilder.STORED_FIELDS_FIELD.match(currentFieldName)) { factory.storedFieldsContext = StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context); - } else if (SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.match(currentFieldName)) { - List fieldDataFields = new ArrayList<>(); + } else if (DocValueFieldsContext.DOCVALUE_FIELDS_FIELD.match(currentFieldName)) { + List fieldDataFields = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - fieldDataFields.add(parser.text()); - } else { - throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING - + "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); - } + fieldDataFields.add(DocValueFieldsContext.Field.fromXContent(parser, context)); } factory.fieldDataFields(fieldDataFields); } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index 6a41cc97f8ec5..509e5a2e0de9d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -47,14 +47,15 @@ public class TopHitsAggregatorFactory extends AggregatorFactory sort; private final HighlightBuilder highlightBuilder; private final StoredFieldsContext storedFieldsContext; - private final List docValueFields; + private final List docValueFields; private final List scriptFields; private final FetchSourceContext fetchSourceContext; public TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean trackScores, Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, - List docValueFields, List scriptFields, FetchSourceContext fetchSourceContext, - SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) + List docValueFields, List scriptFields, + FetchSourceContext fetchSourceContext, SearchContext context, AggregatorFactory parent, + AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, context, parent, subFactories, metaData); this.from = from; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java index a3c7012f7cde1..db7151e2ea1a4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java @@ -81,7 +81,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th boolean hasValue = !(Double.isInfinite(value) || Double.isNaN(value)); builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); if (hasValue && format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java index 9c9da2f26bd53..985e5e2129630 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java @@ -100,7 +100,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th boolean hasValue = !Double.isInfinite(value); builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null); if (hasValue && format != DocValueFormat.RAW) { - builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value)); + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); } builder.startArray("keys"); for (String key : keys) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java index 375011c4e8e18..276a69972f365 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java @@ -82,7 +82,7 @@ public double percentile(double percent) throws IllegalArgumentException { @Override public String percentileAsString(double percent) { - return format.format(percentile(percent)); + return format.format(percentile(percent)).toString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java index db56f0f7c6f0f..d5f2b534e4769 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java @@ -85,7 +85,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th boolean hasValue = !(Double.isInfinite(normalizedValue()) || Double.isNaN(normalizedValue())); builder.field("normalized_value", hasValue ? normalizedValue() : null); if (hasValue && format != DocValueFormat.RAW) { - builder.field("normalized_value_as_string", format.format(normalizedValue())); + builder.field("normalized_value_as_string", format.format(normalizedValue()).toString()); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 37d7eb5b02756..5a44abf3ab0fb 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -44,6 +44,7 @@ import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; @@ -61,6 +62,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * A search source builder allowing to easily build search source. Simple @@ -85,7 +87,6 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ public static final ParseField _SOURCE_FIELD = new ParseField("_source"); public static final ParseField FIELDS_FIELD = new ParseField("fields"); public static final ParseField STORED_FIELDS_FIELD = new ParseField("stored_fields"); - public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields", "fielddata_fields"); public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields"); public static final ParseField SCRIPT_FIELD = new ParseField("script"); public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure"); @@ -151,7 +152,7 @@ public static HighlightBuilder highlight() { private int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER; private StoredFieldsContext storedFieldsContext; - private List docValueFields; + private List docValueFields; private List scriptFields; private FetchSourceContext fetchSourceContext; @@ -186,7 +187,21 @@ public SearchSourceBuilder(StreamInput in) throws IOException { aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new); explain = in.readOptionalBoolean(); fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new); - docValueFields = (List) in.readGenericValue(); + if (in.getVersion().before(Version.V_5_5_0_UNRELEASED)) { + List fieldNameList = (List) in.readGenericValue(); + if (fieldNameList == null) { + docValueFields = null; + } else { + docValueFields = new ArrayList<>(); + for (String name : fieldNameList) { + docValueFields.add(new DocValueFieldsContext.Field(name, null)); + } + } + } else if (in.readBoolean()) { + docValueFields = in.readList(DocValueFieldsContext.Field::new); + } else { + docValueFields = null; + } storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new); from = in.readVInt(); highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new); @@ -230,7 +245,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(aggregations); out.writeOptionalBoolean(explain); out.writeOptionalWriteable(fetchSourceContext); - out.writeGenericValue(docValueFields); + if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) { + out.writeBoolean(docValueFields != null); + if (docValueFields != null) { + out.writeList(docValueFields); + } + } else { + out.writeGenericValue(docValueFields == null ? null : docValueFields.stream().map( + DocValueFieldsContext.Field::getName).collect(Collectors.toList())); + } out.writeOptionalWriteable(storedFieldsContext); out.writeVInt(from); out.writeOptionalWriteable(highlightBuilder); @@ -740,7 +763,7 @@ public SearchSourceBuilder fieldDataField(String name) { if (docValueFields == null) { docValueFields = new ArrayList<>(); } - docValueFields.add(name); + docValueFields.add(new DocValueFieldsContext.Field(name, null)); return this; } @@ -750,7 +773,7 @@ public SearchSourceBuilder fieldDataField(String name) { * @deprecated Use {@link SearchSourceBuilder#docValueFields()} instead. */ @Deprecated - public List fieldDataFields() { + public List fieldDataFields() { return docValueFields; } @@ -758,7 +781,7 @@ public List fieldDataFields() { /** * Gets the docvalue fields. */ - public List docValueFields() { + public List docValueFields() { return docValueFields; } @@ -767,10 +790,20 @@ public List docValueFields() { * search request. */ public SearchSourceBuilder docValueField(String name) { + return docValueField(name, null); + } + + /** + * Adds a field to load from the docvalue and return as part of the + * search request. + * @param name name of the field + * @param format how to format the field + */ + public SearchSourceBuilder docValueField(String name, String format) { if (docValueFields == null) { docValueFields = new ArrayList<>(); } - docValueFields.add(name); + docValueFields.add(new DocValueFieldsContext.Field(name, format)); return this; } @@ -1044,15 +1077,10 @@ public void parseXContent(QueryParseContext context) throws IOException { } else if (token == XContentParser.Token.START_ARRAY) { if (STORED_FIELDS_FIELD.match(currentFieldName)) { storedFieldsContext = StoredFieldsContext.fromXContent(STORED_FIELDS_FIELD.getPreferredName(), context); - } else if (DOCVALUE_FIELDS_FIELD.match(currentFieldName)) { + } else if (DocValueFieldsContext.DOCVALUE_FIELDS_FIELD.match(currentFieldName)) { docValueFields = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - docValueFields.add(parser.text()); - } else { - throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + - "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); - } + docValueFields.add(DocValueFieldsContext.Field.fromXContent(parser, context)); } } else if (INDICES_BOOST_FIELD.match(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { @@ -1152,9 +1180,9 @@ public void innerToXContent(XContentBuilder builder, Params params) throws IOExc } if (docValueFields != null) { - builder.startArray(DOCVALUE_FIELDS_FIELD.getPreferredName()); - for (String fieldDataField : docValueFields) { - builder.value(fieldDataField); + builder.startArray(DocValueFieldsContext.DOCVALUE_FIELDS_FIELD.getPreferredName()); + for (DocValueFieldsContext.Field docValueField : docValueFields) { + docValueField.toXContent(builder); } builder.endArray(); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java index 325d28e459282..b9d4be690327e 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java @@ -18,23 +18,137 @@ */ package org.elasticsearch.search.fetch.subphase; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; + +import java.io.IOException; import java.util.List; +import java.util.Objects; /** * All the required context to pull a field from the doc values. */ public class DocValueFieldsContext { - private final List fields; + public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields", "fielddata_fields"); + public static final ParseField DOCVALUE_FIELD_NAME = new ParseField("name"); + public static final ParseField DOCVALUE_FIELD_FORMAT = new ParseField("format"); + + public static class Field implements Writeable { + + public static Field fromXContent(XContentParser parser, QueryParseContext context) throws IOException { + if (parser.currentToken().isValue()) { + return new Field(parser.text(), null); + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + String name = null; + String format = null; + String currentFieldName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (DOCVALUE_FIELD_NAME.match(currentFieldName)) { + name = parser.text(); + } else if (DOCVALUE_FIELD_FORMAT.match(currentFieldName)) { + format = parser.textOrNull(); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown property under [" + + DOCVALUE_FIELDS_FIELD.getPreferredName() + "] : [" + currentFieldName + + "]", parser.getTokenLocation()); + } + } + if (name == null) { + throw new ParsingException(parser.getTokenLocation(), "Missing name for [" + + DOCVALUE_FIELDS_FIELD.getPreferredName() + "]", parser.getTokenLocation()); + } + return new Field(name, format); + } else { + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + + "] or [" + XContentParser.Token.START_OBJECT + "] in [" + DOCVALUE_FIELDS_FIELD.getPreferredName() + + "] but found [" + parser.currentToken() + "]", parser.getTokenLocation()); + } + } + + private final String name; + private final String format; + + public Field(String name, String format) { + this.name = Objects.requireNonNull(name); + this.format = format; + } + + public Field(StreamInput in) throws IOException { + this(in.readString(), in.readOptionalString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeOptionalString(format); + } + + @Override + public String toString() { + return "Field(name=" + name + ", format=" + format + ")"; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Field other = (Field) obj; + return Objects.equals(name, other.name) && Objects.equals(format, other.format); + } + + @Override + public int hashCode() { + return Objects.hash(name, format); + } + + public void toXContent(XContentBuilder builder) throws IOException { + if (format == null) { + builder.value(name); + } else { + builder.startObject(); + builder.field(DOCVALUE_FIELD_NAME.getPreferredName(), getName()); + builder.field(DOCVALUE_FIELD_FORMAT.getPreferredName(), getFormat()); + builder.endObject(); + } + } + + /** + * Return the name of the field to return. + */ + public String getName() { + return name; + } + + /** + * Return the format specification describing how the field should be + * formatted. {@code null} means that the field defaults should be used. + */ + public String getFormat() { + return format; + } + } + + private final List fields; - public DocValueFieldsContext(List fields) { + public DocValueFieldsContext(List fields) { this.fields = fields; } /** * Returns the required docvalue fields */ - public List fields() { + public List fields() { return this.fields; } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index 42cee23d390cf..102232fd581bf 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -18,9 +18,15 @@ */ package org.elasticsearch.search.fetch.subphase; -import org.elasticsearch.index.fielddata.AtomicFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; @@ -29,6 +35,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; /** * Query sub phase which pulls data from doc values @@ -37,36 +44,81 @@ */ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { + // TODO: Remove in 7.0 + private static final String USE_DEFAULT_FORMAT = "use_field_mapping"; + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DocValueFieldsFetchSubPhase.class)); + @Override public void hitExecute(SearchContext context, HitContext hitContext) throws IOException { if (context.collapse() != null) { // retrieve the `doc_value` associated with the collapse field String name = context.collapse().getFieldType().name(); if (context.docValueFieldsContext() == null) { - context.docValueFieldsContext(new DocValueFieldsContext(Collections.singletonList(name))); + context.docValueFieldsContext(new DocValueFieldsContext( + Collections.singletonList(new DocValueFieldsContext.Field(name, null)))); } else if (context.docValueFieldsContext().fields().contains(name) == false) { - context.docValueFieldsContext().fields().add(name); + context.docValueFieldsContext().fields().add(new DocValueFieldsContext.Field(name, null)); } } if (context.docValueFieldsContext() == null) { return; } - for (String field : context.docValueFieldsContext().fields()) { + for (DocValueFieldsContext.Field field : context.docValueFieldsContext().fields()) { if (hitContext.hit().fieldsOrNull() == null) { hitContext.hit().fields(new HashMap<>(2)); } - SearchHitField hitField = hitContext.hit().getFields().get(field); + SearchHitField hitField = hitContext.hit().getFields().get(field.getName()); if (hitField == null) { - hitField = new SearchHitField(field, new ArrayList<>(2)); - hitContext.hit().getFields().put(field, hitField); + hitField = new SearchHitField(field.getName(), new ArrayList<>(2)); + hitContext.hit().getFields().put(field.getName(), hitField); } - MappedFieldType fieldType = context.mapperService().fullName(field); + MappedFieldType fieldType = context.mapperService().fullName(field.getName()); if (fieldType != null) { - /* Because this is called once per document we end up creating a new ScriptDocValues for every document which is important - * because the values inside ScriptDocValues might be reused for different documents (Dates do this). */ - AtomicFieldData data = context.fieldData().getForField(fieldType).load(hitContext.readerContext()); - ScriptDocValues values = data.getScriptValues(); - values.setNextDocId(hitContext.docId()); + List values = Collections.emptyList(); + String formatName = field.getFormat(); + if (USE_DEFAULT_FORMAT.equals(formatName)) { + // 5.0..5.4 did not format doc values fields, so we exposed the ability to + // use the format associated with the field with a special format name + // `use_field_mapping`, which we are keeping in 6.x to ease the transition from + // 5.x to 6.x + DEPRECATION_LOGGER.deprecated("Format [{}] is deprecated, just omit the format or set it to null in order to use " + + "the field defaults", USE_DEFAULT_FORMAT); + formatName = null; + } + final DocValueFormat format = fieldType.docValueFormat(formatName, null); + final IndexFieldData fieldData = context.fieldData().getForField(fieldType); + if (fieldData instanceof IndexNumericFieldData) { + IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData; + if (numericFieldData.getNumericType().isFloatingPoint()) { + SortedNumericDoubleValues dv = numericFieldData.load(hitContext.readerContext()).getDoubleValues(); + if (dv.advanceExact(hitContext.docId())) { + final int count = dv.docValueCount(); + values = new ArrayList<>(count); + for (int i = 0; i < count; ++i) { + values.add(format.format(dv.nextValue())); + } + } + } else { + SortedNumericDocValues dv = numericFieldData.load(hitContext.readerContext()).getLongValues(); + if (dv.advanceExact(hitContext.docId())) { + final int count = dv.docValueCount(); + values = new ArrayList<>(count); + for (int i = 0; i < count; ++i) { + values.add(format.format(dv.nextValue())); + } + } + } + } else { + SortedBinaryDocValues dv = fieldData.load(hitContext.readerContext()).getBytesValues(); + if (dv.advanceExact(hitContext.docId())) { + final int count = dv.docValueCount(); + values = new ArrayList<>(count); + for (int i = 0; i < count; ++i) { + values.add(format.format(dv.nextValue())); + } + } + } + hitField.getValues().addAll(values); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java index c6590daea3ab5..a290c737df903 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java @@ -18,8 +18,10 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.DocValueFormat; public class BinaryFieldTypeTests extends FieldTypeTestCase { @@ -27,4 +29,11 @@ public class BinaryFieldTypeTests extends FieldTypeTestCase { protected MappedFieldType createDefaultFieldType() { return new BinaryFieldMapper.BinaryFieldType(); } + + public void testValueFormat() { + MappedFieldType ft = createDefaultFieldType(); + DocValueFormat format = ft.docValueFormat(null, null); + assertEquals("ACoB", format.format(new BytesRef(new byte[] {0, 42, 1}))); + assertEquals(new BytesRef(new byte[] {0, 42, 1}), format.parseBytesRef("ACoB")); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index d119a27f22eb5..8621e7758383c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -36,8 +36,8 @@ public void setupProperties() { public void testValueFormat() { MappedFieldType ft = createDefaultFieldType(); - assertEquals("false", ft.docValueFormat(null, null).format(0)); - assertEquals("true", ft.docValueFormat(null, null).format(1)); + assertEquals(false, ft.docValueFormat(null, null).format(0)); + assertEquals(true, ft.docValueFormat(null, null).format(1)); } public void testValueForSearch() { diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 6de9cbab62071..5f78f94367280 100644 --- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; @@ -303,7 +304,7 @@ public static InnerHitBuilder randomInnerHits(boolean recursive, boolean include if (randomBoolean()) { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); } - innerHits.setDocValueFields(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); + innerHits.setDocValueFields(randomListStuff(16, () -> new DocValueFieldsContext.Field(randomAlphaOfLengthBetween(1, 16), null))); // Random script fields deduped on their field name. Map scriptFields = new HashMap<>(); for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) { @@ -367,7 +368,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> { if (randomBoolean()) { copy.setDocValueFields(randomValueOtherThan(copy.getDocValueFields(), () -> { - return randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16)); + return randomListStuff(16, () -> new DocValueFieldsContext.Field(randomAlphaOfLengthBetween(1, 16), null)); })); } else { copy.addDocValueField(randomAlphaOfLengthBetween(1, 16)); diff --git a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index 7bf5308eb635e..a217b86745733 100644 --- a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -44,6 +44,7 @@ public void testSerialization() throws Exception { entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH)); entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP)); entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW)); + entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY)); NamedWriteableRegistry registry = new NamedWriteableRegistry(entries); BytesStreamOutput out = new BytesStreamOutput(); @@ -85,20 +86,38 @@ public void testSerialization() throws Exception { } public void testRawFormat() { - assertEquals("0", DocValueFormat.RAW.format(0)); - assertEquals("-1", DocValueFormat.RAW.format(-1)); - assertEquals("1", DocValueFormat.RAW.format(1)); + assertEquals(0L, DocValueFormat.RAW.format(0)); + assertEquals(-1L, DocValueFormat.RAW.format(-1)); + assertEquals(1L, DocValueFormat.RAW.format(1)); - assertEquals("0.0", DocValueFormat.RAW.format(0d)); - assertEquals("0.5", DocValueFormat.RAW.format(.5d)); - assertEquals("-1.0", DocValueFormat.RAW.format(-1d)); + assertEquals(0d, DocValueFormat.RAW.format(0d)); + assertEquals(0.5, DocValueFormat.RAW.format(.5d)); + assertEquals(-1d, DocValueFormat.RAW.format(-1d)); assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc"))); } + public void testDecimalFormat() { + DocValueFormat format = new DocValueFormat.Decimal("00.0"); + assertEquals("01.0", format.format(1L)); + assertEquals(1, format.parseLong("01.0", randomBoolean(), () -> 42L)); + assertEquals("01.4", format.format(1.43)); + assertEquals(1.4, format.parseDouble("01.4", randomBoolean(), () -> 42L), 0d); + + format = new DocValueFormat.Decimal("###.##"); + assertEquals("0", format.format(0.0d)); + assertEquals("1", format.format(1d)); + format = new DocValueFormat.Decimal("000.000"); + assertEquals("-000.500", format.format(-0.5)); + format = new DocValueFormat.Decimal("###,###.###"); + assertEquals("0.86", format.format(0.8598023539251286d)); + format = new DocValueFormat.Decimal("###,###.###"); + assertEquals("859,802.354", format.format(0.8598023539251286d * 1_000_000)); + } + public void testBooleanFormat() { - assertEquals("false", DocValueFormat.BOOLEAN.format(0)); - assertEquals("true", DocValueFormat.BOOLEAN.format(1)); + assertEquals(false, DocValueFormat.BOOLEAN.format(0)); + assertEquals(true, DocValueFormat.BOOLEAN.format(1)); } public void testIpFormat() { @@ -108,18 +127,6 @@ public void testIpFormat() { DocValueFormat.IP.format(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))))); } - public void testDecimalFormat() { - DocValueFormat formatter = new DocValueFormat.Decimal("###.##"); - assertEquals("0", formatter.format(0.0d)); - assertEquals("1", formatter.format(1d)); - formatter = new DocValueFormat.Decimal("000.000"); - assertEquals("-000.500", formatter.format(-0.5)); - formatter = new DocValueFormat.Decimal("###,###.###"); - assertEquals("0.86", formatter.format(0.8598023539251286d)); - formatter = new DocValueFormat.Decimal("###,###.###"); - assertEquals("859,802.354", formatter.format(0.8598023539251286d * 1_000_000)); - } - public void testRawParse() { assertEquals(-1L, DocValueFormat.RAW.parseLong("-1", randomBoolean(), null)); assertEquals(1L, DocValueFormat.RAW.parseLong("1", randomBoolean(), null)); @@ -169,4 +176,9 @@ public void testDecimalParse() { assertEquals(0.859d, parser.parseDouble("0.859", true, null), 0.0d); assertEquals(0.8598023539251286d, parser.parseDouble("0.8598023539251286", true, null), 0.0d); } + + public void testBinaryFormat() { + assertEquals("ACoB", DocValueFormat.BINARY.format(new BytesRef(new byte[] {0, 42, 1}))); + assertEquals(new BytesRef(new byte[] {0, 42, 1}), DocValueFormat.BINARY.parseBytesRef("ACoB")); + } } diff --git a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 428ac63eaefa7..0ed68aeabd795 100644 --- a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -724,7 +724,7 @@ public void testSingleValueFieldDatatField() throws ExecutionException, Interrup assertThat(fields.get("test_field").getValue(), equalTo("foobar")); } - public void testFieldsPulledFromFieldData() throws Exception { + public void testFieldsPulledFromDocValues() throws Exception { createIndex("test"); String mapping = XContentFactory.jsonBuilder() @@ -767,6 +767,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .endObject() .startObject("binary_field") .field("type", "binary") + .field("doc_values", true) .endObject() .startObject("ip_field") .field("type", "ip") @@ -790,6 +791,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .field("double_field", 6.0d) .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date)) .field("boolean_field", true) + .field("binary_field", "ACoB") // this is base 64 .field("ip_field", "::1") .endObject()).execute().actionGet(); @@ -806,6 +808,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .addDocValueField("double_field") .addDocValueField("date_field") .addDocValueField("boolean_field") + .addDocValueField("binary_field") .addDocValueField("ip_field"); SearchResponse searchResponse = builder.execute().actionGet(); @@ -814,7 +817,7 @@ public void testFieldsPulledFromFieldData() throws Exception { Set fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet()); assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field", "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field", - "ip_field"))); + "binary_field", "ip_field"))); assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1")); assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2")); @@ -822,10 +825,81 @@ public void testFieldsPulledFromFieldData() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), equalTo(date)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), equalTo("2012-03-22T00:00:00.000Z")); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("ACoB")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); + + builder = client().prepareSearch().setQuery(matchAllQuery()) + .addDocValueField("text_field", null) + .addDocValueField("keyword_field", null) + .addDocValueField("byte_field", "#.0") + .addDocValueField("short_field", "#.0") + .addDocValueField("integer_field", "#.0") + .addDocValueField("long_field", "#.0") + .addDocValueField("float_field", "#.0") + .addDocValueField("double_field", "#.0") + .addDocValueField("date_field", "epoch_millis") + .addDocValueField("boolean_field", null) + .addDocValueField("binary_field", null) + .addDocValueField("ip_field", null); + searchResponse = builder.execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet()); + assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field", + "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field", + "binary_field", "ip_field"))); + + assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1.0")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2.0")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("integer_field").getValue(), equalTo("3.0")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo("4.0")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo("5.0")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo("6.0")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), equalTo("1332374400000")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("ACoB")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); + + builder = client().prepareSearch().setQuery(matchAllQuery()) + .addDocValueField("text_field", "use_field_mapping") + .addDocValueField("keyword_field", "use_field_mapping") + .addDocValueField("byte_field", "use_field_mapping") + .addDocValueField("short_field", "use_field_mapping") + .addDocValueField("integer_field", "use_field_mapping") + .addDocValueField("long_field", "use_field_mapping") + .addDocValueField("float_field", "use_field_mapping") + .addDocValueField("double_field", "use_field_mapping") + .addDocValueField("date_field", "use_field_mapping") + .addDocValueField("boolean_field", "use_field_mapping") + .addDocValueField("binary_field", "use_field_mapping") + .addDocValueField("ip_field", "use_field_mapping"); + searchResponse = builder.execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet()); + assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field", + "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field", + "binary_field", "ip_field"))); + + assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("integer_field").getValue(), equalTo((Object) 3L)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), equalTo("2012-03-22T00:00:00.000Z")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("ACoB")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); } diff --git a/docs/reference/migration/migrate_6_0/search.asciidoc b/docs/reference/migration/migrate_6_0/search.asciidoc index 7b080eeb3be83..ca22e63976af0 100644 --- a/docs/reference/migration/migrate_6_0/search.asciidoc +++ b/docs/reference/migration/migrate_6_0/search.asciidoc @@ -88,3 +88,34 @@ produces. BM25 is recommended instead. See https://issues.apache.org/jira/browse/LUCENE-7347[`LUCENE-7347`] for more information. + +==== `docvalue_fields` + +The `docvalue_fields` now returns a formatted value for the field. This is +likely mostly an issue if you used `docvalue_fields` to return dates as a number +of milliseconds since Epoch. It is still possible to format dates as the number +of milliseconds since Epoch by using the `epoch_millis` format: + +[source,js] +-------------------------------------------------- +GET _search +{ + "docvalue_fields" : [ { "name": "my_date", "format": "epoch_millis" } ] +} +-------------------------------------------------- +// CONSOLE + +Also note that in order to ease the transition from 5.x to 6.x, a special +`format` called `use_field_mapping` makes Elasticsearch 5.x format fields like +in 6.x. This way, you can deal with the change in format while you are still on +5.x and not worry about it when doing the upgrade. Just stop using this special +`format` as the transition is finished since it will be removed in 7.0. + +[source,js] +-------------------------------------------------- +GET _search +{ + "docvalue_fields" : [ { "name": "my_date", "format": "use_field_mapping" } ] +} +-------------------------------------------------- +// CONSOLE diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc index b4d2493d8536d..daacca6ddb3eb 100644 --- a/docs/reference/search/request/docvalue-fields.asciidoc +++ b/docs/reference/search/request/docvalue-fields.asciidoc @@ -21,3 +21,67 @@ Doc value fields can work on fields that are not stored. Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption. +For date fields, it is also possible to specify the format that should be used to format the date, eg. + +[source,js] +-------------------------------------------------- +PUT index +{ + "mappings": { + "type": { + "properties": { + "my_date": { + "type": "date" + } + } + } + } +} + +PUT index/type/1?refresh=true +{ + "my_date": "2016-10-28T11:55:21.945Z" +} + +GET index/_search +{ + "docvalue_fields" : [ { "name": "my_date", "format": "epoch_second" } ] +} +-------------------------------------------------- +// CONSOLE + +which returns: + +[source,js] +-------------------------------------------------- +{ + "took": 36, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 1.0, + "hits": [ + { + "_index": "index", + "_type": "type", + "_id": "1", + "_score": 1.0, + "_source": { + "my_date": "2016-10-28T11:55:21.945Z" + }, + "fields": { + "my_date": [ + "1477655721" + ] + } + } + ] + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"took": 36,/"took": "$body.took",/] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml index c8f6871295e6a..8595853db7592 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml @@ -132,8 +132,18 @@ setup: - is_true: hits.hits.0._source --- -"fielddata_fields": +"docvalue_fields": - do: search: docvalue_fields: [ "count" ] - match: { hits.hits.0.fields.count: [1] } + + +--- +"docvalue_fields with format": + - do: + search: + body: + docvalue_fields: [ { name: "count", format: "#.0"} ] + - match: { hits.hits.0.fields.count: ["1.0"] } + From d35c9794de072bc74a15c5d356a45915a2fc488b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 21 Apr 2017 10:15:03 +0200 Subject: [PATCH 2/4] Fix UOE. --- .../subphase/DocValueFieldsFetchSubPhase.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index 102232fd581bf..16b33360c9124 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -50,20 +50,16 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) throws IOException { + List docValueFields = Collections.emptyList(); + if (context.docValueFieldsContext() != null) { + docValueFields = context.docValueFieldsContext().fields(); + } if (context.collapse() != null) { - // retrieve the `doc_value` associated with the collapse field String name = context.collapse().getFieldType().name(); - if (context.docValueFieldsContext() == null) { - context.docValueFieldsContext(new DocValueFieldsContext( - Collections.singletonList(new DocValueFieldsContext.Field(name, null)))); - } else if (context.docValueFieldsContext().fields().contains(name) == false) { - context.docValueFieldsContext().fields().add(new DocValueFieldsContext.Field(name, null)); - } - } - if (context.docValueFieldsContext() == null) { - return; + docValueFields = new ArrayList<>(docValueFields); + docValueFields.add(new DocValueFieldsContext.Field(name, null)); } - for (DocValueFieldsContext.Field field : context.docValueFieldsContext().fields()) { + for (DocValueFieldsContext.Field field : docValueFields) { if (hitContext.hit().fieldsOrNull() == null) { hitContext.hit().fields(new HashMap<>(2)); } From b5b56f604a00488a2672d27c9f264e40a79417d4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 21 Apr 2017 10:21:22 +0200 Subject: [PATCH 3/4] Add skip. --- .../rest-api-spec/test/search/10_source_filtering.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml index 8595853db7592..ceab50f9d6d3b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml @@ -141,6 +141,8 @@ setup: --- "docvalue_fields with format": + version: " - 5.4.99" + reason: feature was added in 5.5.0 - do: search: body: From 39e70307a91b27a449b0d162d6ed05b7b0299651 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 21 Apr 2017 10:24:07 +0200 Subject: [PATCH 4/4] iter --- .../rest-api-spec/test/search/10_source_filtering.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml index ceab50f9d6d3b..eee8abc26f99a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml @@ -141,7 +141,8 @@ setup: --- "docvalue_fields with format": - version: " - 5.4.99" + - skip: + version: " - 5.4.99" reason: feature was added in 5.5.0 - do: search: