Skip to content

Format doc values fields. #22146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Format doc values fields. #22146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1490e1
Format doc values fields.
jpountz Dec 13, 2016
d35c979
Fix UOE.
jpountz Apr 21, 2017
b5b56f6
Add skip.
jpountz Apr 21, 2017
39e7030
iter
jpountz Apr 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio
}
}
if (options.getDocValueFields() != null) {
options.getDocValueFields().forEach(groupSource::docValueField);
options.getDocValueFields().forEach(field -> groupSource.docValueField(field.getName(), field.getFormat()));
}
if (options.getStoredFieldsContext() != null && options.getStoredFieldsContext().fieldNames() != null) {
options.getStoredFieldsContext().fieldNames().forEach(groupSource::storedField);
Expand Down
14 changes: 13 additions & 1 deletion core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ public SearchRequestBuilder setFetchSource(@Nullable String[] includes, @Nullabl

/**
* Adds a docvalue based field to load and return. The field does not have to be stored,
* but its recommended to use non analyzed or numeric fields.
* but its recommended to use non analyzed fields.
*
* @param name The field to get from the docvalue
*/
Expand All @@ -293,6 +293,18 @@ public SearchRequestBuilder addDocValueField(String name) {
return this;
}

/**
* Adds a docvalue based field to load and return. The field does not have to be stored,
* but its recommended to use non analyzed fields.
*
* @param name The field to get from the docvalue
* @param format How to format the field, {@code null} to use defaults.
*/
public SearchRequestBuilder addDocValueField(String name, String format) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we're usually OK with making breaking changes to the Transport API in a minor release it is probably ok to add the format parameter to this method if you think it makes sense.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know Kibana uses this feature to retrieve dates in millis since Epoch, so with this option they will be able to pass format: epoch_millis when we format docvalue fields. I also plan to use it for the transition period: users will be able in 5.x to enable the 6.x behaviour by using the special format use_field_defaults.

sourceBuilder().docValueField(name, format);
return this;
}

/**
* Adds a stored field to load and return (note, it must be stored) as part of the search request.
*/
Expand Down
7 changes: 7 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.Base64;
Expand Down Expand Up @@ -120,6 +122,11 @@ public BytesReference valueForDisplay(Object value) {
return bytes;
}

@Override
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
return DocValueFormat.BINARY;
}

@Override
public IndexFieldData.Builder fielddataBuilder() {
failIfNoDocValues();
Expand Down
65 changes: 48 additions & 17 deletions core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.elasticsearch.Version;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
Expand Down Expand Up @@ -54,6 +55,7 @@
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

import static org.elasticsearch.common.xcontent.XContentParser.Token.END_OBJECT;

Expand Down Expand Up @@ -81,7 +83,9 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
"if the field is not stored");
}, SearchSourceBuilder.FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY);
PARSER.declareStringArray(InnerHitBuilder::setDocValueFields, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD);
PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields,
DocValueFieldsContext.Field::fromXContent,
DocValueFieldsContext.DOCVALUE_FIELDS_FIELD);
PARSER.declareField((p, i, c) -> {
try {
Set<ScriptField> scriptFields = new HashSet<>();
Expand Down Expand Up @@ -144,7 +148,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
private StoredFieldsContext storedFieldsContext;
private QueryBuilder query = DEFAULT_INNER_HIT_QUERY;
private List<SortBuilder<?>> sorts;
private List<String> docValueFields;
private List<DocValueFieldsContext.Field> docValueFields;
private Set<ScriptField> scriptFields;
private HighlightBuilder highlightBuilder;
private FetchSourceContext fetchSourceContext;
Expand Down Expand Up @@ -221,7 +225,21 @@ public InnerHitBuilder(StreamInput in) throws IOException {
version = in.readBoolean();
trackScores = in.readBoolean();
storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new);
docValueFields = (List<String>) in.readGenericValue();
if (in.getVersion().before(Version.V_5_5_0_UNRELEASED)) {
List<String> fieldNameList = (List<String>) in.readGenericValue();
if (fieldNameList == null) {
docValueFields = null;
} else {
docValueFields = new ArrayList<>();
for (String name : fieldNameList) {
docValueFields.add(new DocValueFieldsContext.Field(name, null));
}
}
} else if (in.readBoolean()) {
docValueFields = in.readList(DocValueFieldsContext.Field::new);
} else {
docValueFields = null;
}
if (in.readBoolean()) {
int size = in.readVInt();
scriptFields = new HashSet<>(size);
Expand Down Expand Up @@ -262,7 +280,15 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(version);
out.writeBoolean(trackScores);
out.writeOptionalWriteable(storedFieldsContext);
out.writeGenericValue(docValueFields);
if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
out.writeBoolean(docValueFields != null);
if (docValueFields != null) {
out.writeList(docValueFields);
}
} else {
out.writeGenericValue(docValueFields == null ? null : docValueFields.stream().map(
DocValueFieldsContext.Field::getName).collect(Collectors.toList()));
}
boolean hasScriptFields = scriptFields != null;
out.writeBoolean(hasScriptFields);
if (hasScriptFields) {
Expand Down Expand Up @@ -413,7 +439,7 @@ public InnerHitBuilder setStoredFieldNames(List<String> fieldNames) {
* @deprecated Use {@link InnerHitBuilder#getDocValueFields()} instead.
*/
@Deprecated
public List<String> getFieldDataFields() {
public List<DocValueFieldsContext.Field> getFieldDataFields() {
return docValueFields;
}

Expand All @@ -423,7 +449,7 @@ public List<String> getFieldDataFields() {
* @deprecated Use {@link InnerHitBuilder#setDocValueFields(List)} instead.
*/
@Deprecated
public InnerHitBuilder setFieldDataFields(List<String> fieldDataFields) {
public InnerHitBuilder setFieldDataFields(List<DocValueFieldsContext.Field> fieldDataFields) {
this.docValueFields = fieldDataFields;
return this;
}
Expand All @@ -435,24 +461,20 @@ public InnerHitBuilder setFieldDataFields(List<String> fieldDataFields) {
*/
@Deprecated
public InnerHitBuilder addFieldDataField(String field) {
if (docValueFields == null) {
docValueFields = new ArrayList<>();
}
docValueFields.add(field);
return this;
return addDocValueField(field);
}

/**
* Gets the docvalue fields.
*/
public List<String> getDocValueFields() {
public List<DocValueFieldsContext.Field> getDocValueFields() {
return docValueFields;
}

/**
* Sets the stored fields to load from the docvalue and return.
*/
public InnerHitBuilder setDocValueFields(List<String> docValueFields) {
public InnerHitBuilder setDocValueFields(List<DocValueFieldsContext.Field> docValueFields) {
this.docValueFields = docValueFields;
return this;
}
Expand All @@ -461,10 +483,19 @@ public InnerHitBuilder setDocValueFields(List<String> docValueFields) {
* Adds a field to load from the docvalue and return.
*/
public InnerHitBuilder addDocValueField(String field) {
return addDocValueField(field, null);
}

/**
* Adds a field to load from the docvalue and return.
* @param name name of the field
* @param format how to format the field, or {@code null} to use the defaults
*/
public InnerHitBuilder addDocValueField(String name, @Nullable String format) {
if (docValueFields == null) {
docValueFields = new ArrayList<>();
}
docValueFields.add(field);
docValueFields.add(new DocValueFieldsContext.Field(name, format));
return this;
}

Expand Down Expand Up @@ -673,9 +704,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
storedFieldsContext.toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder);
}
if (docValueFields != null) {
builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
for (String fieldDataField : docValueFields) {
builder.value(fieldDataField);
builder.startArray(DocValueFieldsContext.DOCVALUE_FIELDS_FIELD.getPreferredName());
for (DocValueFieldsContext.Field fieldDataField : docValueFields) {
fieldDataField.toXContent(builder);
}
builder.endArray();
}
Expand Down
Loading