Skip to content

Commit c214c1e

Browse files
jtibshiranijtibshirani
committed
Return null_value when the source contains a 'null' for the field. (#58623)
This PR adds a version of `XContentMapValues.extractValue` that accepts a default value to return in place of 'null'. It then uses this method when looking up source values to return the configured `null_value` instead of 'null' when retrieving fields.
1 parent 236dc0d commit c214c1e

File tree

22 files changed

+299
-34
lines changed

22 files changed

+299
-34
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,11 @@ protected ScaledFloatFieldMapper clone() {
354354
return (ScaledFloatFieldMapper) super.clone();
355355
}
356356

357+
@Override
358+
protected Double nullValue() {
359+
return nullValue;
360+
}
361+
357362
@Override
358363
protected void parseCreateField(ParseContext context) throws IOException {
359364

@@ -478,7 +483,16 @@ protected Double parseSourceValue(Object value, String format) {
478483
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
479484
}
480485

481-
double doubleValue = objectToDouble(value);
486+
double doubleValue;
487+
if (value.equals("")) {
488+
if (nullValue == null) {
489+
return null;
490+
}
491+
doubleValue = nullValue;
492+
} else {
493+
doubleValue = objectToDouble(value);
494+
}
495+
482496
double scalingFactor = fieldType().getScalingFactor();
483497
return Math.round(doubleValue * scalingFactor) / scalingFactor;
484498
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.index.IndexService;
3333
import org.elasticsearch.index.mapper.MapperService.MergeReason;
3434
import org.elasticsearch.plugins.Plugin;
35+
import org.elasticsearch.search.lookup.SourceLookup;
3536
import org.elasticsearch.test.InternalSettingsPlugin;
3637
import org.junit.Before;
3738

@@ -405,11 +406,22 @@ public void testMeta() throws Exception {
405406
public void testParseSourceValue() {
406407
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
407408
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
409+
408410
ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field")
409411
.scalingFactor(100)
410412
.build(context);
411-
412413
assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001);
413414
assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001);
415+
assertNull(mapper.parseSourceValue("", null));
416+
417+
ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field")
418+
.scalingFactor(100)
419+
.nullValue(2.71)
420+
.build(context);
421+
assertEquals(2.71, nullValueMapper.parseSourceValue("", null), 0.00001);
422+
423+
SourceLookup sourceLookup = new SourceLookup();
424+
sourceLookup.setSource(Collections.singletonMap("field", null));
425+
assertEquals(List.of(2.71), nullValueMapper.lookupValues(sourceLookup, null));
414426
}
415427
}

‎plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

+5
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,11 @@ protected String contentType() {
577577
return CONTENT_TYPE;
578578
}
579579

580+
@Override
581+
protected String nullValue() {
582+
return nullValue;
583+
}
584+
580585
@Override
581586
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
582587
ICUCollationKeywordFieldMapper icuMergeWith = (ICUCollationKeywordFieldMapper) other;

‎plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,15 @@
3838
import org.elasticsearch.index.mapper.MapperService.MergeReason;
3939
import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
4040
import org.elasticsearch.plugins.Plugin;
41+
import org.elasticsearch.search.lookup.SourceLookup;
4142
import org.elasticsearch.test.InternalSettingsPlugin;
4243
import org.junit.Before;
4344

4445
import java.io.IOException;
4546
import java.util.Arrays;
4647
import java.util.Collection;
48+
import java.util.Collections;
49+
import java.util.List;
4750
import java.util.Set;
4851

4952
import static org.hamcrest.Matchers.containsString;
@@ -489,9 +492,8 @@ public void testUpdateIgnoreAbove() throws IOException {
489492
public void testParseSourceValue() {
490493
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
491494
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
492-
ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context);
493495

494-
assertEquals("value", mapper.parseSourceValue("value", null));
496+
ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context);
495497
assertEquals("42", mapper.parseSourceValue(42L, null));
496498
assertEquals("true", mapper.parseSourceValue(true, null));
497499

@@ -501,5 +503,12 @@ public void testParseSourceValue() {
501503
assertNull(ignoreAboveMapper.parseSourceValue("value", null));
502504
assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null));
503505
assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null));
506+
507+
ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field")
508+
.nullValue("NULL")
509+
.build(context);
510+
SourceLookup sourceLookup = new SourceLookup();
511+
sourceLookup.setSource(Collections.singletonMap("field", null));
512+
assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null));
504513
}
505514
}

‎server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

+55-19
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,16 @@ private static void extractRawValues(List values, List<Object> part, String[] pa
9797
}
9898
}
9999

100+
/**
101+
* For the provided path, return its value in the xContent map.
102+
*
103+
* Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values
104+
* can be returned.
105+
*
106+
* @param path the value's path in the map.
107+
*
108+
* @return the value associated with the path in the map or 'null' if the path does not exist.
109+
*/
100110
public static Object extractValue(String path, Map<?, ?> map) {
101111
return extractValue(map, path.split("\\."));
102112
}
@@ -105,19 +115,51 @@ public static Object extractValue(Map<?, ?> map, String... pathElements) {
105115
if (pathElements.length == 0) {
106116
return null;
107117
}
108-
return extractValue(pathElements, 0, map);
118+
return XContentMapValues.extractValue(pathElements, 0, map, null);
109119
}
110120

111-
@SuppressWarnings({"unchecked"})
112-
private static Object extractValue(String[] pathElements, int index, Object currentValue) {
113-
if (index == pathElements.length) {
114-
return currentValue;
115-
}
116-
if (currentValue == null) {
121+
/**
122+
* For the provided path, return its value in the xContent map.
123+
*
124+
* Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values
125+
* can be returned.
126+
*
127+
* @param path the value's path in the map.
128+
* @param nullValue a value to return if the path exists, but the value is 'null'. This helps
129+
* in distinguishing between a path that doesn't exist vs. a value of 'null'.
130+
*
131+
* @return the value associated with the path in the map or 'null' if the path does not exist.
132+
*/
133+
public static Object extractValue(String path, Map<?, ?> map, Object nullValue) {
134+
String[] pathElements = path.split("\\.");
135+
if (pathElements.length == 0) {
117136
return null;
118137
}
138+
return extractValue(pathElements, 0, map, nullValue);
139+
}
140+
141+
private static Object extractValue(String[] pathElements,
142+
int index,
143+
Object currentValue,
144+
Object nullValue) {
145+
if (currentValue instanceof List) {
146+
List<?> valueList = (List<?>) currentValue;
147+
List<Object> newList = new ArrayList<>(valueList.size());
148+
for (Object o : valueList) {
149+
Object listValue = extractValue(pathElements, index, o, nullValue);
150+
if (listValue != null) {
151+
newList.add(listValue);
152+
}
153+
}
154+
return newList;
155+
}
156+
157+
if (index == pathElements.length) {
158+
return currentValue != null ? currentValue : nullValue;
159+
}
160+
119161
if (currentValue instanceof Map) {
120-
Map map = (Map) currentValue;
162+
Map<?, ?> map = (Map<?, ?>) currentValue;
121163
String key = pathElements[index];
122164
Object mapValue = map.get(key);
123165
int nextIndex = index + 1;
@@ -126,18 +168,12 @@ private static Object extractValue(String[] pathElements, int index, Object curr
126168
mapValue = map.get(key);
127169
nextIndex++;
128170
}
129-
return extractValue(pathElements, nextIndex, mapValue);
130-
}
131-
if (currentValue instanceof List) {
132-
List valueList = (List) currentValue;
133-
List newList = new ArrayList(valueList.size());
134-
for (Object o : valueList) {
135-
Object listValue = extractValue(pathElements, index, o);
136-
if (listValue != null) {
137-
newList.add(listValue);
138-
}
171+
172+
if (map.containsKey(key) == false) {
173+
return null;
139174
}
140-
return newList;
175+
176+
return extractValue(pathElements, nextIndex, mapValue, nullValue);
141177
}
142178
return null;
143179
}

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

+4
Original file line numberDiff line numberDiff line change
@@ -281,4 +281,8 @@ protected String contentType() {
281281
return CONTENT_TYPE;
282282
}
283283

284+
@Override
285+
protected Object nullValue() {
286+
return nullValue;
287+
}
284288
}

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,11 @@ protected DateFieldMapper clone() {
504504
return (DateFieldMapper) super.clone();
505505
}
506506

507+
@Override
508+
protected String nullValue() {
509+
return nullValueAsString;
510+
}
511+
507512
@Override
508513
protected void parseCreateField(ParseContext context) throws IOException {
509514
String dateAsString;
@@ -554,8 +559,8 @@ protected void parseCreateField(ParseContext context) throws IOException {
554559
public String parseSourceValue(Object value, String format) {
555560
String date = value.toString();
556561
long timestamp = fieldType().parse(date);
557-
ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
558562

563+
ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
559564
DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter();
560565
if (format != null) {
561566
dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale());

‎server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,13 @@ public CopyTo copyTo() {
224224
return copyTo;
225225
}
226226

227+
/**
228+
* A value to use in place of a {@code null} value in the document source.
229+
*/
230+
protected Object nullValue() {
231+
return null;
232+
}
233+
227234
/**
228235
* Whether this mapper can handle an array value during document parsing. If true,
229236
* when an array is encountered during parsing, the document parser will pass the
@@ -286,7 +293,7 @@ public void parse(ParseContext context) throws IOException {
286293
* @return a list a standardized field values.
287294
*/
288295
public List<?> lookupValues(SourceLookup lookup, @Nullable String format) {
289-
Object sourceValue = lookup.extractValue(name());
296+
Object sourceValue = lookup.extractValue(name(), nullValue());
290297
if (sourceValue == null) {
291298
return List.of();
292299
}
@@ -338,6 +345,7 @@ protected FieldMapper clone() {
338345
}
339346
}
340347

348+
341349
@Override
342350
public FieldMapper merge(Mapper mergeWith) {
343351
FieldMapper merged = clone();

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,11 @@ protected String contentType() {
348348
return fieldType().typeName();
349349
}
350350

351+
@Override
352+
protected Object nullValue() {
353+
return nullValue;
354+
}
355+
351356
@Override
352357
protected IpFieldMapper clone() {
353358
return (IpFieldMapper) super.clone();
@@ -406,7 +411,12 @@ protected String parseSourceValue(Object value, String format) {
406411
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
407412
}
408413

409-
InetAddress address = InetAddresses.forString(value.toString());
414+
InetAddress address;
415+
if (value instanceof InetAddress) {
416+
address = (InetAddress) value;
417+
} else {
418+
address = InetAddresses.forString(value.toString());
419+
}
410420
return InetAddresses.toAddrString(address);
411421
}
412422

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

+5
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,11 @@ protected String contentType() {
416416
return CONTENT_TYPE;
417417
}
418418

419+
@Override
420+
protected String nullValue() {
421+
return nullValue;
422+
}
423+
419424
@Override
420425
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
421426
KeywordFieldMapper k = (KeywordFieldMapper) other;

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

+10
Original file line numberDiff line numberDiff line change
@@ -1036,6 +1036,11 @@ protected NumberFieldMapper clone() {
10361036
return (NumberFieldMapper) super.clone();
10371037
}
10381038

1039+
@Override
1040+
protected Number nullValue() {
1041+
return nullValue;
1042+
}
1043+
10391044
@Override
10401045
protected void parseCreateField(ParseContext context) throws IOException {
10411046
XContentParser parser = context.parser();
@@ -1090,6 +1095,11 @@ protected Number parseSourceValue(Object value, String format) {
10901095
if (format != null) {
10911096
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
10921097
}
1098+
1099+
if (value.equals("")) {
1100+
return nullValue;
1101+
}
1102+
10931103
return fieldType().type.parse(value, coerce.value());
10941104
}
10951105

‎server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java

+13-4
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.lucene.index.LeafReader;
2222
import org.apache.lucene.index.LeafReaderContext;
2323
import org.elasticsearch.ElasticsearchParseException;
24+
import org.elasticsearch.common.Nullable;
2425
import org.elasticsearch.common.bytes.BytesReference;
2526
import org.elasticsearch.common.collect.Tuple;
2627
import org.elasticsearch.common.xcontent.XContentHelper;
@@ -133,11 +134,19 @@ public List<Object> extractRawValues(String path) {
133134
}
134135

135136
/**
136-
* For the provided path, return its value in the source. Note that in contrast with
137-
* {@link SourceLookup#extractRawValues}, array and object values can be returned.
137+
* For the provided path, return its value in the source.
138+
*
139+
* Note that in contrast with {@link SourceLookup#extractRawValues}, array and object values
140+
* can be returned.
141+
*
142+
* @param path the value's path in the source.
143+
* @param nullValue a value to return if the path exists, but the value is 'null'. This helps
144+
* in distinguishing between a path that doesn't exist vs. a value of 'null'.
145+
*
146+
* @return the value associated with the path in the source or 'null' if the path does not exist.
138147
*/
139-
public Object extractValue(String path) {
140-
return XContentMapValues.extractValue(path, loadSourceIfNeeded());
148+
public Object extractValue(String path, @Nullable Object nullValue) {
149+
return XContentMapValues.extractValue(path, loadSourceIfNeeded(), nullValue);
141150
}
142151

143152
public Object filter(FetchSourceContext context) {

0 commit comments

Comments
 (0)